Coverage for python/lsst/cp/pipe/defects.py: 20%

365 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-02-28 12:47 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

24 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask', 

26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ] 

27 

28import numpy as np 

29 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32 

33from lsstDebug import getDebugFrame 

34import lsst.pex.config as pexConfig 

35 

36import lsst.afw.image as afwImage 

37import lsst.afw.math as afwMath 

38import lsst.afw.detection as afwDetection 

39import lsst.afw.display as afwDisplay 

40from lsst.afw import cameraGeom 

41from lsst.geom import Box2I, Point2I 

42from lsst.meas.algorithms import SourceDetectionTask 

43from lsst.ip.isr import Defects, countMaskedPixels 

44from lsst.pex.exceptions import InvalidParameterError 

45 

46 

47class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("instrument", "exposure", "detector")): 

49 inputExp = cT.Input( 

50 name="defectExps", 

51 doc="Input ISR-processed exposures to measure.", 

52 storageClass="Exposure", 

53 dimensions=("instrument", "detector", "exposure"), 

54 multiple=False 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name='camera', 

58 doc="Camera associated with this exposure.", 

59 storageClass="Camera", 

60 dimensions=("instrument", ), 

61 isCalibration=True, 

62 ) 

63 

64 outputDefects = cT.Output( 

65 name="singleExpDefects", 

66 doc="Output measured defects.", 

67 storageClass="Defects", 

68 dimensions=("instrument", "detector", "exposure"), 

69 ) 

70 

71 

72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=MeasureDefectsConnections): 

74 """Configuration for measuring defects from a list of exposures 

75 """ 

76 

77 thresholdType = pexConfig.ChoiceField( 

78 dtype=str, 

79 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found " 

80 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found " 

81 "in flats, and hot pixels in darks."), 

82 default='STDEV', 

83 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.", 

84 'VALUE': "Use pixel value to determine detection threshold."}, 

85 ) 

86 darkCurrentThreshold = pexConfig.Field( 

87 dtype=float, 

88 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define " 

89 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."), 

90 default=5, 

91 ) 

92 fracThresholdFlat = pexConfig.Field( 

93 dtype=float, 

94 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark " 

95 "pixels in flat images (fraction of the mean value per amplifier)." 

96 "Unused if thresholdType==``STDEV``."), 

97 default=0.8, 

98 ) 

99 nSigmaBright = pexConfig.Field( 

100 dtype=float, 

101 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot " 

102 "pixel detection. The default value was found to be " 

103 "appropriate for some LSST sensors in DM-17490. " 

104 "Unused if thresholdType==``VALUE``"), 

105 default=4.8, 

106 ) 

107 nSigmaDark = pexConfig.Field( 

108 dtype=float, 

109 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel " 

110 "detection. The default value was found to be " 

111 "appropriate for some LSST sensors in DM-17490. " 

112 "Unused if thresholdType==``VALUE``"), 

113 default=-5.0, 

114 ) 

115 nPixBorderUpDown = pexConfig.Field( 

116 dtype=int, 

117 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

118 default=7, 

119 ) 

120 nPixBorderLeftRight = pexConfig.Field( 

121 dtype=int, 

122 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

123 default=7, 

124 ) 

125 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

126 dtype=int, 

127 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

128 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

129 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

130 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

131 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

132 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

133 "'goodPixelColumnGapThreshold' are not marked as bad."), 

134 default=50, 

135 ) 

136 goodPixelColumnGapThreshold = pexConfig.Field( 

137 dtype=int, 

138 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

139 "'badOnAndOffPixelColumnThreshold')."), 

140 default=30, 

141 ) 

142 

143 def validate(self): 

144 super().validate() 

145 if self.nSigmaBright < 0.0: 

146 raise ValueError("nSigmaBright must be above 0.0.") 

147 if self.nSigmaDark > 0.0: 

148 raise ValueError("nSigmaDark must be below 0.0.") 

149 

150 

151class MeasureDefectsTask(pipeBase.PipelineTask): 

152 """Measure the defects from one exposure. 

153 """ 

154 

155 ConfigClass = MeasureDefectsTaskConfig 

156 _DefaultName = 'cpDefectMeasure' 

157 

158 def run(self, inputExp, camera): 

159 """Measure one exposure for defects. 

160 

161 Parameters 

162 ---------- 

163 inputExp : `lsst.afw.image.Exposure` 

164 Exposure to examine. 

165 camera : `lsst.afw.cameraGeom.Camera` 

166 Camera to use for metadata. 

167 

168 Returns 

169 ------- 

170 results : `lsst.pipe.base.Struct` 

171 Results struct containing: 

172 

173 ``outputDefects`` 

174 The defects measured from this exposure 

175 (`lsst.ip.isr.Defects`). 

176 """ 

177 detector = inputExp.getDetector() 

178 try: 

179 filterName = inputExp.getFilter().physicalLabel 

180 except AttributeError: 

181 filterName = None 

182 

183 defects = self._findHotAndColdPixels(inputExp) 

184 

185 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

186 msg = "Found %s defects containing %s pixels in %s" 

187 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

188 

189 defects.updateMetadataFromExposures([inputExp]) 

190 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

191 setCalibId=True, setDate=True, 

192 cpDefectGenImageType=datasetType) 

193 

194 return pipeBase.Struct( 

195 outputDefects=defects, 

196 ) 

197 

198 @staticmethod 

199 def _nPixFromDefects(defects): 

200 """Count pixels in a defect. 

201 

202 Parameters 

203 ---------- 

204 defects : `lsst.ip.isr.Defects` 

205 Defects to measure. 

206 

207 Returns 

208 ------- 

209 nPix : `int` 

210 Number of defect pixels. 

211 """ 

212 nPix = 0 

213 for defect in defects: 

214 nPix += defect.getBBox().getArea() 

215 return nPix 

216 

217 def _findHotAndColdPixels(self, exp): 

218 """Find hot and cold pixels in an image. 

219 

220 Using config-defined thresholds on a per-amp basis, mask 

221 pixels that are nSigma above threshold in dark frames (hot 

222 pixels), or nSigma away from the clipped mean in flats (hot & 

223 cold pixels). 

224 

225 Parameters 

226 ---------- 

227 exp : `lsst.afw.image.exposure.Exposure` 

228 The exposure in which to find defects. 

229 

230 Returns 

231 ------- 

232 defects : `lsst.ip.isr.Defects` 

233 The defects found in the image. 

234 """ 

235 self._setEdgeBits(exp) 

236 maskedIm = exp.maskedImage 

237 

238 # the detection polarity for afwDetection, True for positive, 

239 # False for negative, and therefore True for darks as they only have 

240 # bright pixels, and both for flats, as they have bright and dark pix 

241 footprintList = [] 

242 

243 hotPixelCount = {} 

244 coldPixelCount = {} 

245 

246 for amp in exp.getDetector(): 

247 ampName = amp.getName() 

248 

249 hotPixelCount[ampName] = 0 

250 coldPixelCount[ampName] = 0 

251 

252 ampImg = maskedIm[amp.getBBox()].clone() 

253 

254 # crop ampImage depending on where the amp lies in the image 

255 if self.config.nPixBorderLeftRight: 

256 if ampImg.getX0() == 0: 

257 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

258 else: 

259 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

260 if self.config.nPixBorderUpDown: 

261 if ampImg.getY0() == 0: 

262 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

263 else: 

264 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

265 

266 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

267 continue 

268 

269 # Remove a background estimate 

270 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

271 ampImg -= meanClip 

272 

273 # Determine thresholds 

274 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue() 

275 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

276 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

277 if np.isnan(expTime): 

278 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second", 

279 expTime, ampName, datasetType) 

280 expTime = 1. 

281 thresholdType = self.config.thresholdType 

282 if thresholdType == 'VALUE': 

283 # LCA-128 and eoTest: bright/hot pixels in dark images are 

284 # defined as any pixel with more than 5 e-/s of dark current. 

285 # We scale by the exposure time. 

286 if datasetType.lower() == 'dark': 

287 # hot pixel threshold 

288 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain() 

289 else: 

290 # LCA-128 and eoTest: dark/cold pixels in flat images as 

291 # defined as any pixel with photoresponse <80% of 

292 # the mean (at 500nm). 

293 

294 # We subtracted the mean above, so the threshold will be 

295 # negative cold pixel threshold. 

296 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip 

297 # Find equivalent sigma values. 

298 if stDev == 0.0: 

299 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.", 

300 stDev, ampName, datasetType) 

301 nSigmaList = [np.inf] 

302 else: 

303 nSigmaList = [valueThreshold/stDev] 

304 else: 

305 hotPixelThreshold = self.config.nSigmaBright 

306 coldPixelThreshold = self.config.nSigmaDark 

307 if datasetType.lower() == 'dark': 

308 nSigmaList = [hotPixelThreshold] 

309 valueThreshold = stDev*hotPixelThreshold 

310 else: 

311 nSigmaList = [hotPixelThreshold, coldPixelThreshold] 

312 valueThreshold = [x*stDev for x in nSigmaList] 

313 

314 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel" 

315 "Values (hot and cold pixels thresholds): %s, %s", 

316 datasetType, ampName, thresholdType, nSigmaList, valueThreshold) 

317 mergedSet = None 

318 for sigma in nSigmaList: 

319 nSig = np.abs(sigma) 

320 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

321 polarity = {-1: False, 1: True}[np.sign(sigma)] 

322 

323 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

324 

325 try: 

326 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

327 except InvalidParameterError: 

328 # This occurs if the image sigma value is 0.0. 

329 # Let's mask the whole area. 

330 minValue = np.nanmin(ampImg.image.array) - 1.0 

331 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True) 

332 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

333 

334 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

335 

336 if mergedSet is None: 

337 mergedSet = footprintSet 

338 else: 

339 mergedSet.merge(footprintSet) 

340 

341 if polarity: 

342 # hot pixels 

343 for fp in footprintSet.getFootprints(): 

344 hotPixelCount[ampName] += fp.getArea() 

345 else: 

346 # cold pixels 

347 for fp in footprintSet.getFootprints(): 

348 coldPixelCount[ampName] += fp.getArea() 

349 

350 footprintList += mergedSet.getFootprints() 

351 

352 self.debugView('defectMap', ampImg, 

353 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

354 

355 defects = Defects.fromFootprintList(footprintList) 

356 defects, count = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

357 defects.updateCounters(columns=count, hot=hotPixelCount, cold=coldPixelCount) 

358 

359 return defects 

360 

361 @staticmethod 

362 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

363 """Return the number of non-bad pixels in the image.""" 

364 nPixels = maskedIm.mask.array.size 

365 nBad = countMaskedPixels(maskedIm, badMaskString) 

366 return nPixels - nBad 

367 

368 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

369 """Set edge bits on an exposure or maskedImage. 

370 

371 Raises 

372 ------ 

373 TypeError 

374 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

375 """ 

376 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

377 mi = exposureOrMaskedImage.maskedImage 

378 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

379 mi = exposureOrMaskedImage 

380 else: 

381 t = type(exposureOrMaskedImage) 

382 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

383 

384 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

385 if self.config.nPixBorderLeftRight: 

386 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

387 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

388 if self.config.nPixBorderUpDown: 

389 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

390 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

391 

392 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

393 """Mask blocks in a column if there are on-and-off bad pixels 

394 

395 If there's a column with on and off bad pixels, mask all the 

396 pixels in between, except if there is a large enough gap of 

397 consecutive good pixels between two bad pixels in the column. 

398 

399 Parameters 

400 ---------- 

401 defects : `lsst.ip.isr.Defects` 

402 The defects found in the image so far 

403 

404 Returns 

405 ------- 

406 defects : `lsst.ip.isr.Defects` 

407 If the number of bad pixels in a column is not larger or 

408 equal than self.config.badPixelColumnThreshold, the input 

409 list is returned. Otherwise, the defects list returned 

410 will include boxes that mask blocks of on-and-of pixels. 

411 badColumnCount : `int` 

412 Number of bad columns masked. 

413 """ 

414 badColumnCount = 0 

415 # Get the (x, y) values of each bad pixel in amp. 

416 coordinates = [] 

417 for defect in defects: 

418 bbox = defect.getBBox() 

419 x0, y0 = bbox.getMinX(), bbox.getMinY() 

420 deltaX0, deltaY0 = bbox.getDimensions() 

421 for j in np.arange(y0, y0+deltaY0): 

422 for i in np.arange(x0, x0 + deltaX0): 

423 coordinates.append((i, j)) 

424 

425 x, y = [], [] 

426 for coordinatePair in coordinates: 

427 x.append(coordinatePair[0]) 

428 y.append(coordinatePair[1]) 

429 

430 x = np.array(x) 

431 y = np.array(y) 

432 # Find the defects with same "x" (vertical) coordinate (column). 

433 unique, counts = np.unique(x, return_counts=True) 

434 multipleX = [] 

435 for (a, b) in zip(unique, counts): 

436 if b >= self.config.badOnAndOffPixelColumnThreshold: 

437 multipleX.append(a) 

438 if len(multipleX) != 0: 

439 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

440 badColumnCount += 1 

441 

442 return defects, badColumnCount 

443 

444 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

445 """Mask blocks in a column if number of on-and-off bad pixels is above 

446 threshold. 

447 

448 This function is called if the number of on-and-off bad pixels 

449 in a column is larger or equal than 

450 self.config.badOnAndOffPixelColumnThreshold. 

451 

452 Parameters 

453 --------- 

454 x : `list` 

455 Lower left x coordinate of defect box. x coordinate is 

456 along the short axis if amp. 

457 y : `list` 

458 Lower left y coordinate of defect box. x coordinate is 

459 along the long axis if amp. 

460 multipleX : list 

461 List of x coordinates in amp. with multiple bad pixels 

462 (i.e., columns with defects). 

463 defects : `lsst.ip.isr.Defects` 

464 The defcts found in the image so far 

465 

466 Returns 

467 ------- 

468 defects : `lsst.ip.isr.Defects` 

469 The defects list returned that will include boxes that 

470 mask blocks of on-and-of pixels. 

471 """ 

472 with defects.bulk_update(): 

473 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

474 for x0 in multipleX: 

475 index = np.where(x == x0) 

476 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

477 multipleY.sort() # Ensure that the y values are sorted to look for gaps. 

478 minY, maxY = np.min(multipleY), np.max(multipleY) 

479 # Next few lines: don't mask pixels in column if gap 

480 # of good pixels between two consecutive bad pixels is 

481 # larger or equal than 'goodPixelColumnGapThreshold'. 

482 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

483 if len(diffIndex) != 0: 

484 limits = [minY] # put the minimum first 

485 for gapIndex in diffIndex: 

486 limits.append(multipleY[gapIndex]) 

487 limits.append(multipleY[gapIndex+1]) 

488 limits.append(maxY) # maximum last 

489 for i in np.arange(0, len(limits)-1, 2): 

490 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

491 defects.append(s) 

492 else: # No gap is large enough 

493 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

494 defects.append(s) 

495 return defects 

496 

497 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

498 """Plot the defects found by the task. 

499 

500 Parameters 

501 ---------- 

502 stepname : `str` 

503 Debug frame to request. 

504 ampImage : `lsst.afw.image.MaskedImage` 

505 Amplifier image to display. 

506 defects : `lsst.ip.isr.Defects` 

507 The defects to plot. 

508 detector : `lsst.afw.cameraGeom.Detector` 

509 Detector holding camera geometry. 

510 """ 

511 frame = getDebugFrame(self._display, stepname) 

512 if frame: 

513 disp = afwDisplay.Display(frame=frame) 

514 disp.scale('asinh', 'zscale') 

515 disp.setMaskTransparency(80) 

516 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

517 

518 maskedIm = ampImage.clone() 

519 defects.maskPixels(maskedIm, "BAD") 

520 

521 mpDict = maskedIm.mask.getMaskPlaneDict() 

522 for plane in mpDict.keys(): 

523 if plane in ['BAD']: 

524 continue 

525 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

526 

527 disp.setImageColormap('gray') 

528 disp.mtv(maskedIm) 

529 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

530 prompt = "Press Enter to continue [c]... " 

531 while True: 

532 ans = input(prompt).lower() 

533 if ans in ('', 'c', ): 

534 break 

535 

536 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

537 """Make a histogram of the distribution of pixel values for 

538 each amp. 

539 

540 The main image data histogram is plotted in blue. Edge 

541 pixels, if masked, are in red. Note that masked edge pixels 

542 do not contribute to the underflow and overflow numbers. 

543 

544 Note that this currently only supports the 16-amp LSST 

545 detectors. 

546 

547 Parameters 

548 ---------- 

549 stepname : `str` 

550 Debug frame to request. 

551 ampImage : `lsst.afw.image.MaskedImage` 

552 Amplifier image to display. 

553 nSigmaUsed : `float` 

554 The number of sigma used for detection 

555 exp : `lsst.afw.image.exposure.Exposure` 

556 The exposure in which the defects were found. 

557 """ 

558 frame = getDebugFrame(self._display, stepname) 

559 if frame: 

560 import matplotlib.pyplot as plt 

561 

562 detector = exp.getDetector() 

563 nX = np.floor(np.sqrt(len(detector))) 

564 nY = len(detector) // nX 

565 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

566 

567 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

568 

569 for (amp, a) in zip(reversed(detector), ax.flatten()): 

570 mi = exp.maskedImage[amp.getBBox()] 

571 

572 # normalize by expTime as we plot in ADU/s and don't 

573 # always work with master calibs 

574 mi.image.array /= expTime 

575 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

576 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

577 # Get array of pixels 

578 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

579 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

580 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

581 

582 thrUpper = mean + nSigmaUsed*sigma 

583 thrLower = mean - nSigmaUsed*sigma 

584 

585 nRight = len(imgData[imgData > thrUpper]) 

586 nLeft = len(imgData[imgData < thrLower]) 

587 

588 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

589 leftEdge = mean - nsig * nSigmaUsed*sigma 

590 rightEdge = mean + nsig * nSigmaUsed*sigma 

591 nbins = np.linspace(leftEdge, rightEdge, 1000) 

592 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

593 lw=1, edgecolor='red') 

594 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

595 lw=3, edgecolor='blue') 

596 

597 # Report number of entries in over- and under-flow 

598 # bins, i.e. off the edges of the histogram 

599 nOverflow = len(imgData[imgData > rightEdge]) 

600 nUnderflow = len(imgData[imgData < leftEdge]) 

601 

602 # Put v-lines and textboxes in 

603 a.axvline(thrUpper, c='k') 

604 a.axvline(thrLower, c='k') 

605 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

606 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

607 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

608 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

609 

610 # set axis limits and scales 

611 a.set_ylim([1., 1.7*np.max(y)]) 

612 lPlot, rPlot = a.get_xlim() 

613 a.set_xlim(np.array([lPlot, rPlot])) 

614 a.set_yscale('log') 

615 a.set_xlabel("ADU/s") 

616 fig.show() 

617 prompt = "Press Enter or c to continue [chp]..." 

618 while True: 

619 ans = input(prompt).lower() 

620 if ans in ("", " ", "c",): 

621 break 

622 elif ans in ("p", ): 

623 import pdb 

624 pdb.set_trace() 

625 elif ans in ("h", ): 

626 print("[h]elp [c]ontinue [p]db") 

627 plt.close() 

628 

629 

630class MeasureDefectsCombinedConnections(pipeBase.PipelineTaskConnections, 

631 dimensions=("instrument", "detector")): 

632 inputExp = cT.Input( 

633 name="dark", 

634 doc="Input ISR-processed combined exposure to measure.", 

635 storageClass="ExposureF", 

636 dimensions=("instrument", "detector"), 

637 multiple=False, 

638 isCalibration=True, 

639 ) 

640 camera = cT.PrerequisiteInput( 

641 name='camera', 

642 doc="Camera associated with this exposure.", 

643 storageClass="Camera", 

644 dimensions=("instrument", ), 

645 isCalibration=True, 

646 ) 

647 

648 outputDefects = cT.Output( 

649 name="cpPartialDefectsFromDarkCombined", 

650 doc="Output measured defects.", 

651 storageClass="Defects", 

652 dimensions=("instrument", "detector"), 

653 ) 

654 

655 

656class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig, 

657 pipelineConnections=MeasureDefectsCombinedConnections): 

658 """Configuration for measuring defects from combined exposures. 

659 """ 

660 pass 

661 

662 

663class MeasureDefectsCombinedTask(MeasureDefectsTask): 

664 """Task to measure defects in combined images.""" 

665 

666 ConfigClass = MeasureDefectsCombinedTaskConfig 

667 _DefaultName = "cpDefectMeasureCombined" 

668 

669 

670class MeasureDefectsCombinedWithFilterConnections(pipeBase.PipelineTaskConnections, 

671 dimensions=("instrument", "detector", "physical_filter")): 

672 """Task to measure defects in combined flats under a certain filter.""" 

673 inputExp = cT.Input( 

674 name="flat", 

675 doc="Input ISR-processed combined exposure to measure.", 

676 storageClass="ExposureF", 

677 dimensions=("instrument", "detector", "physical_filter"), 

678 multiple=False, 

679 isCalibration=True, 

680 ) 

681 camera = cT.PrerequisiteInput( 

682 name='camera', 

683 doc="Camera associated with this exposure.", 

684 storageClass="Camera", 

685 dimensions=("instrument", ), 

686 isCalibration=True, 

687 ) 

688 

689 outputDefects = cT.Output( 

690 name="cpPartialDefectsFromFlatCombinedWithFilter", 

691 doc="Output measured defects.", 

692 storageClass="Defects", 

693 dimensions=("instrument", "detector", "physical_filter"), 

694 ) 

695 

696 

697class MeasureDefectsCombinedWithFilterTaskConfig( 

698 MeasureDefectsTaskConfig, 

699 pipelineConnections=MeasureDefectsCombinedWithFilterConnections): 

700 """Configuration for measuring defects from combined exposures. 

701 """ 

702 pass 

703 

704 

705class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask): 

706 """Task to measure defects in combined images.""" 

707 

708 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig 

709 _DefaultName = "cpDefectMeasureWithFilterCombined" 

710 

711 

712class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

713 dimensions=("instrument", "detector")): 

714 inputDefects = cT.Input( 

715 name="singleExpDefects", 

716 doc="Measured defect lists.", 

717 storageClass="Defects", 

718 dimensions=("instrument", "detector", "exposure",), 

719 multiple=True, 

720 ) 

721 camera = cT.PrerequisiteInput( 

722 name='camera', 

723 doc="Camera associated with these defects.", 

724 storageClass="Camera", 

725 dimensions=("instrument", ), 

726 isCalibration=True, 

727 ) 

728 

729 mergedDefects = cT.Output( 

730 name="defects", 

731 doc="Final merged defects.", 

732 storageClass="Defects", 

733 dimensions=("instrument", "detector"), 

734 multiple=False, 

735 isCalibration=True, 

736 ) 

737 

738 

739class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

740 pipelineConnections=MergeDefectsConnections): 

741 """Configuration for merging single exposure defects. 

742 """ 

743 

744 assertSameRun = pexConfig.Field( 

745 dtype=bool, 

746 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

747 "if the run key isn't found."), 

748 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

749 ) 

750 ignoreFilters = pexConfig.Field( 

751 dtype=bool, 

752 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

753 " images. Allows mixing of filters in the input flats. Set to False if you think" 

754 " your defects might be chromatic and want to have registry support for varying" 

755 " defects with respect to filter."), 

756 default=True, 

757 ) 

758 nullFilterName = pexConfig.Field( 

759 dtype=str, 

760 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

761 default="NONE", 

762 ) 

763 combinationMode = pexConfig.ChoiceField( 

764 doc="Which types of defects to identify", 

765 dtype=str, 

766 default="FRACTION", 

767 allowed={ 

768 "AND": "Logical AND the pixels found in each visit to form set ", 

769 "OR": "Logical OR the pixels found in each visit to form set ", 

770 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

771 } 

772 ) 

773 combinationFraction = pexConfig.RangeField( 

774 dtype=float, 

775 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

776 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

777 " mode AND to require pixel to appear in all images."), 

778 default=0.7, 

779 min=0, 

780 max=1, 

781 ) 

782 edgesAsDefects = pexConfig.Field( 

783 dtype=bool, 

784 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

785 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

786 " defect will be located there."), 

787 default=False, 

788 ) 

789 

790 

791class MergeDefectsTask(pipeBase.PipelineTask): 

792 """Merge the defects from multiple exposures. 

793 """ 

794 

795 ConfigClass = MergeDefectsTaskConfig 

796 _DefaultName = 'cpDefectMerge' 

797 

798 def run(self, inputDefects, camera): 

799 """Merge a list of single defects to find the common defect regions. 

800 

801 Parameters 

802 ---------- 

803 inputDefects : `list` [`lsst.ip.isr.Defects`] 

804 Partial defects from a single exposure. 

805 camera : `lsst.afw.cameraGeom.Camera` 

806 Camera to use for metadata. 

807 

808 Returns 

809 ------- 

810 results : `lsst.pipe.base.Struct` 

811 Results struct containing: 

812 

813 ``mergedDefects`` 

814 The defects merged from the input lists 

815 (`lsst.ip.isr.Defects`). 

816 """ 

817 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

818 if detectorId is None: 

819 raise RuntimeError("Cannot identify detector id.") 

820 detector = camera[detectorId] 

821 

822 imageTypes = set() 

823 for inDefect in inputDefects: 

824 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

825 imageTypes.add(imageType) 

826 

827 # Determine common defect pixels separately for each input image type. 

828 splitDefects = list() 

829 for imageType in imageTypes: 

830 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

831 count = 0 

832 for inDefect in inputDefects: 

833 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

834 count += 1 

835 for defect in inDefect: 

836 sumImage.image[defect.getBBox()] += 1.0 

837 sumImage /= count 

838 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

839 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

840 

841 if self.config.combinationMode == 'AND': 

842 threshold = 1.0 

843 elif self.config.combinationMode == 'OR': 

844 threshold = 0.0 

845 elif self.config.combinationMode == 'FRACTION': 

846 threshold = self.config.combinationFraction 

847 else: 

848 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

849 indices = np.where(sumImage.getImage().getArray() > threshold) 

850 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

851 sumImage.getMask().getArray()[indices] |= BADBIT 

852 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

853 partialDefect = Defects.fromMask(sumImage, 'BAD') 

854 splitDefects.append(partialDefect) 

855 

856 # Do final combination of separate image types 

857 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

858 for inDefect in splitDefects: 

859 for defect in inDefect: 

860 finalImage.image[defect.getBBox()] += 1 

861 finalImage /= len(splitDefects) 

862 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) 

863 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) 

864 

865 # This combination is the OR of all image types 

866 threshold = 0.0 

867 indices = np.where(finalImage.getImage().getArray() > threshold) 

868 BADBIT = finalImage.getMask().getPlaneBitMask('BAD') 

869 finalImage.getMask().getArray()[indices] |= BADBIT 

870 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) 

871 

872 if self.config.edgesAsDefects: 

873 self.log.info("Masking edge pixels as defects.") 

874 # Do the same as IsrTask.maskEdges() 

875 box = detector.getBBox() 

876 subImage = finalImage[box] 

877 box.grow(-self.nPixBorder) 

878 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

879 

880 merged = Defects.fromMask(finalImage, 'BAD') 

881 merged.updateMetadataFromExposures(inputDefects) 

882 merged.updateMetadata(camera=camera, detector=detector, filterName=None, 

883 setCalibId=True, setDate=True) 

884 

885 return pipeBase.Struct( 

886 mergedDefects=merged, 

887 ) 

888 

889# Subclass the MergeDefects task to reduce the input dimensions 

890# from ("instrument", "detector", "exposure") to 

891# ("instrument", "detector"). 

892 

893 

894class MergeDefectsCombinedConnections(pipeBase.PipelineTaskConnections, 

895 dimensions=("instrument", "detector")): 

896 inputDarkDefects = cT.Input( 

897 name="cpPartialDefectsFromDarkCombined", 

898 doc="Measured defect lists.", 

899 storageClass="Defects", 

900 dimensions=("instrument", "detector",), 

901 multiple=True, 

902 ) 

903 inputFlatDefects = cT.Input( 

904 name="cpPartialDefectsFromFlatCombinedWithFilter", 

905 doc="Additional measured defect lists.", 

906 storageClass="Defects", 

907 dimensions=("instrument", "detector", "physical_filter"), 

908 multiple=True, 

909 ) 

910 camera = cT.PrerequisiteInput( 

911 name='camera', 

912 doc="Camera associated with these defects.", 

913 storageClass="Camera", 

914 dimensions=("instrument", ), 

915 isCalibration=True, 

916 ) 

917 

918 mergedDefects = cT.Output( 

919 name="defects", 

920 doc="Final merged defects.", 

921 storageClass="Defects", 

922 dimensions=("instrument", "detector"), 

923 multiple=False, 

924 isCalibration=True, 

925 ) 

926 

927 

928class MergeDefectsCombinedTaskConfig(MergeDefectsTaskConfig, 

929 pipelineConnections=MergeDefectsCombinedConnections): 

930 """Configuration for merging defects from combined exposure. 

931 """ 

932 def validate(self): 

933 super().validate() 

934 if self.combinationMode != 'OR': 

935 raise ValueError("combinationMode must be 'OR'") 

936 

937 

938class MergeDefectsCombinedTask(MergeDefectsTask): 

939 """Task to measure defects in combined images.""" 

940 

941 ConfigClass = MergeDefectsCombinedTaskConfig 

942 _DefaultName = "cpDefectMergeCombined" 

943 

944 @staticmethod 

945 def chooseBest(inputs): 

946 """Select the input with the most exposures used.""" 

947 best = 0 

948 if len(inputs) > 1: 

949 nInput = 0 

950 for num, exp in enumerate(inputs): 

951 # This technically overcounts by a factor of 3. 

952 N = len([k for k, v in exp.getMetadata().toDict().items() if "CPP_INPUT_" in k]) 

953 if N > nInput: 

954 best = num 

955 nInput = N 

956 return inputs[best] 

957 

958 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

959 inputs = butlerQC.get(inputRefs) 

960 # Turn inputFlatDefects and inputDarkDefects into a list which 

961 # is what MergeDefectsTask expects. If there are multiple, 

962 # use the one with the most inputs. 

963 tempList = [self.chooseBest(inputs['inputFlatDefects']), 

964 self.chooseBest(inputs['inputDarkDefects'])] 

965 

966 # Rename inputDefects 

967 inputsCombined = {'inputDefects': tempList, 'camera': inputs['camera']} 

968 

969 outputs = super().run(**inputsCombined) 

970 butlerQC.put(outputs, outputRefs)