Coverage for python/lsst/cp/pipe/defects.py: 18%

432 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 02:24 -0700

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

24 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

25 'MeasureDefectsCombinedTaskConfig', 'MeasureDefectsCombinedTask', 

26 'MergeDefectsCombinedTaskConfig', 'MergeDefectsCombinedTask', ] 

27 

28import numpy as np 

29 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32 

33from lsstDebug import getDebugFrame 

34import lsst.pex.config as pexConfig 

35 

36import lsst.afw.image as afwImage 

37import lsst.afw.math as afwMath 

38import lsst.afw.detection as afwDetection 

39import lsst.afw.display as afwDisplay 

40from lsst.afw import cameraGeom 

41from lsst.geom import Box2I, Point2I, Extent2I 

42from lsst.meas.algorithms import SourceDetectionTask 

43from lsst.ip.isr import Defects, countMaskedPixels 

44from lsst.pex.exceptions import InvalidParameterError 

45 

46 

47class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

48 dimensions=("instrument", "exposure", "detector")): 

49 inputExp = cT.Input( 

50 name="defectExps", 

51 doc="Input ISR-processed exposures to measure.", 

52 storageClass="Exposure", 

53 dimensions=("instrument", "detector", "exposure"), 

54 multiple=False 

55 ) 

56 camera = cT.PrerequisiteInput( 

57 name='camera', 

58 doc="Camera associated with this exposure.", 

59 storageClass="Camera", 

60 dimensions=("instrument", ), 

61 isCalibration=True, 

62 ) 

63 

64 outputDefects = cT.Output( 

65 name="singleExpDefects", 

66 doc="Output measured defects.", 

67 storageClass="Defects", 

68 dimensions=("instrument", "detector", "exposure"), 

69 ) 

70 

71 

72class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

73 pipelineConnections=MeasureDefectsConnections): 

74 """Configuration for measuring defects from a list of exposures 

75 """ 

76 

77 thresholdType = pexConfig.ChoiceField( 

78 dtype=str, 

79 doc=("Defects threshold type: ``STDEV`` or ``VALUE``. If ``VALUE``, cold pixels will be found " 

80 "in flats, and hot pixels in darks. If ``STDEV``, cold and hot pixels will be found " 

81 "in flats, and hot pixels in darks."), 

82 default='STDEV', 

83 allowed={'STDEV': "Use a multiple of the image standard deviation to determine detection threshold.", 

84 'VALUE': "Use pixel value to determine detection threshold."}, 

85 ) 

86 darkCurrentThreshold = pexConfig.Field( 

87 dtype=float, 

88 doc=("If thresholdType=``VALUE``, dark current threshold (in e-/sec) to define " 

89 "hot/bright pixels in dark images. Unused if thresholdType==``STDEV``."), 

90 default=5, 

91 ) 

92 biasThreshold = pexConfig.Field( 

93 dtype=float, 

94 doc=("If thresholdType==``VALUE``, bias threshold (in ADU) to define " 

95 "hot/bright pixels in bias frame. Unused if thresholdType==``STDEV``."), 

96 default=1000.0, 

97 ) 

98 fracThresholdFlat = pexConfig.Field( 

99 dtype=float, 

100 doc=("If thresholdType=``VALUE``, fractional threshold to define cold/dark " 

101 "pixels in flat images (fraction of the mean value per amplifier)." 

102 "Unused if thresholdType==``STDEV``."), 

103 default=0.8, 

104 ) 

105 nSigmaBright = pexConfig.Field( 

106 dtype=float, 

107 doc=("If thresholdType=``STDEV``, number of sigma above mean for bright/hot " 

108 "pixel detection. The default value was found to be " 

109 "appropriate for some LSST sensors in DM-17490. " 

110 "Unused if thresholdType==``VALUE``"), 

111 default=4.8, 

112 ) 

113 nSigmaDark = pexConfig.Field( 

114 dtype=float, 

115 doc=("If thresholdType=``STDEV``, number of sigma below mean for dark/cold pixel " 

116 "detection. The default value was found to be " 

117 "appropriate for some LSST sensors in DM-17490. " 

118 "Unused if thresholdType==``VALUE``"), 

119 default=-5.0, 

120 ) 

121 nPixBorderUpDown = pexConfig.Field( 

122 dtype=int, 

123 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

124 default=0, 

125 ) 

126 nPixBorderLeftRight = pexConfig.Field( 

127 dtype=int, 

128 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

129 default=0, 

130 ) 

131 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

132 dtype=int, 

133 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) " 

134 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the " 

135 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long " 

136 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all " 

137 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive " 

138 "non-bad pixels in BPC, an exception to the above is made and those consecutive " 

139 "'goodPixelColumnGapThreshold' are not marked as bad."), 

140 default=50, 

141 ) 

142 goodPixelColumnGapThreshold = pexConfig.Field( 

143 dtype=int, 

144 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see " 

145 "'badOnAndOffPixelColumnThreshold')."), 

146 default=30, 

147 ) 

148 badPixelsToFillColumnThreshold = pexConfig.Field( 

149 dtype=float, 

150 doc=("If the number of bad pixels in an amplifier column is above this threshold " 

151 "then the full amplifier column will be marked bad. This operation is performed after " 

152 "any merging of blinking columns performed with badOnAndOffPixelColumnThreshold. If this" 

153 "value is less than 0 then no bad column filling will be performed."), 

154 default=-1, 

155 ) 

156 saturatedColumnMask = pexConfig.Field( 

157 dtype=str, 

158 default="SAT", 

159 doc="Saturated mask plane for dilation.", 

160 ) 

161 saturatedColumnDilationRadius = pexConfig.Field( 

162 dtype=int, 

163 doc=("Dilation radius (along rows) to use to expand saturated columns " 

164 "to mitigate glow."), 

165 default=0, 

166 ) 

167 saturatedPixelsToFillColumnThreshold = pexConfig.Field( 

168 dtype=int, 

169 doc=("If the number of saturated pixels in an amplifier column is above this threshold " 

170 "then the full amplifier column will be marked bad. If this value is less than 0" 

171 "then no saturated column filling will be performed."), 

172 default=-1, 

173 ) 

174 

175 def validate(self): 

176 super().validate() 

177 if self.nSigmaBright < 0.0: 

178 raise ValueError("nSigmaBright must be above 0.0.") 

179 if self.nSigmaDark > 0.0: 

180 raise ValueError("nSigmaDark must be below 0.0.") 

181 

182 

183class MeasureDefectsTask(pipeBase.PipelineTask): 

184 """Measure the defects from one exposure. 

185 """ 

186 

187 ConfigClass = MeasureDefectsTaskConfig 

188 _DefaultName = 'cpDefectMeasure' 

189 

190 def run(self, inputExp, camera): 

191 """Measure one exposure for defects. 

192 

193 Parameters 

194 ---------- 

195 inputExp : `lsst.afw.image.Exposure` 

196 Exposure to examine. 

197 camera : `lsst.afw.cameraGeom.Camera` 

198 Camera to use for metadata. 

199 

200 Returns 

201 ------- 

202 results : `lsst.pipe.base.Struct` 

203 Results struct containing: 

204 

205 ``outputDefects`` 

206 The defects measured from this exposure 

207 (`lsst.ip.isr.Defects`). 

208 """ 

209 detector = inputExp.getDetector() 

210 try: 

211 filterName = inputExp.getFilter().physicalLabel 

212 except AttributeError: 

213 filterName = None 

214 

215 defects = self._findHotAndColdPixels(inputExp) 

216 

217 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

218 msg = "Found %s defects containing %s pixels in %s" 

219 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

220 

221 defects.updateMetadataFromExposures([inputExp]) 

222 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

223 setCalibId=True, setDate=True, 

224 cpDefectGenImageType=datasetType) 

225 

226 return pipeBase.Struct( 

227 outputDefects=defects, 

228 ) 

229 

230 @staticmethod 

231 def _nPixFromDefects(defects): 

232 """Count pixels in a defect. 

233 

234 Parameters 

235 ---------- 

236 defects : `lsst.ip.isr.Defects` 

237 Defects to measure. 

238 

239 Returns 

240 ------- 

241 nPix : `int` 

242 Number of defect pixels. 

243 """ 

244 nPix = 0 

245 for defect in defects: 

246 nPix += defect.getBBox().getArea() 

247 return nPix 

248 

249 def _findHotAndColdPixels(self, exp): 

250 """Find hot and cold pixels in an image. 

251 

252 Using config-defined thresholds on a per-amp basis, mask 

253 pixels that are nSigma above threshold in dark frames (hot 

254 pixels), or nSigma away from the clipped mean in flats (hot & 

255 cold pixels). 

256 

257 Parameters 

258 ---------- 

259 exp : `lsst.afw.image.exposure.Exposure` 

260 The exposure in which to find defects. 

261 

262 Returns 

263 ------- 

264 defects : `lsst.ip.isr.Defects` 

265 The defects found in the image. 

266 """ 

267 self._setEdgeBits(exp) 

268 maskedIm = exp.maskedImage 

269 

270 # the detection polarity for afwDetection, True for positive, 

271 # False for negative, and therefore True for darks as they only have 

272 # bright pixels, and both for flats, as they have bright and dark pix 

273 footprintList = [] 

274 

275 hotPixelCount = {} 

276 coldPixelCount = {} 

277 

278 for amp in exp.getDetector(): 

279 ampName = amp.getName() 

280 

281 hotPixelCount[ampName] = 0 

282 coldPixelCount[ampName] = 0 

283 

284 ampImg = maskedIm[amp.getBBox()].clone() 

285 

286 # crop ampImage depending on where the amp lies in the image 

287 if self.config.nPixBorderLeftRight: 

288 if ampImg.getX0() == 0: 

289 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

290 else: 

291 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

292 if self.config.nPixBorderUpDown: 

293 if ampImg.getY0() == 0: 

294 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

295 else: 

296 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

297 

298 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

299 continue 

300 

301 # Remove a background estimate 

302 meanClip = afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

303 ampImg -= meanClip 

304 

305 # Determine thresholds 

306 stDev = afwMath.makeStatistics(ampImg, afwMath.STDEVCLIP, ).getValue() 

307 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

308 datasetType = exp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

309 if np.isnan(expTime): 

310 self.log.warning("expTime=%s for AMP %s in %s. Setting expTime to 1 second", 

311 expTime, ampName, datasetType) 

312 expTime = 1. 

313 thresholdType = self.config.thresholdType 

314 if thresholdType == 'VALUE': 

315 # LCA-128 and eoTest: bright/hot pixels in dark images are 

316 # defined as any pixel with more than 5 e-/s of dark current. 

317 # We scale by the exposure time. 

318 if datasetType.lower() == 'dark': 

319 # hot pixel threshold 

320 valueThreshold = self.config.darkCurrentThreshold*expTime/amp.getGain() 

321 elif datasetType.lower() == 'bias': 

322 # hot pixel threshold, no exposure time. 

323 valueThreshold = self.config.biasThreshold 

324 else: 

325 # LCA-128 and eoTest: dark/cold pixels in flat images as 

326 # defined as any pixel with photoresponse <80% of 

327 # the mean (at 500nm). 

328 

329 # We subtracted the mean above, so the threshold will be 

330 # negative cold pixel threshold. 

331 valueThreshold = (self.config.fracThresholdFlat-1)*meanClip 

332 # Find equivalent sigma values. 

333 if stDev == 0.0: 

334 self.log.warning("stDev=%s for AMP %s in %s. Setting nSigma to inf.", 

335 stDev, ampName, datasetType) 

336 nSigmaList = [np.inf] 

337 else: 

338 nSigmaList = [valueThreshold/stDev] 

339 else: 

340 hotPixelThreshold = self.config.nSigmaBright 

341 coldPixelThreshold = self.config.nSigmaDark 

342 if datasetType.lower() == 'dark': 

343 nSigmaList = [hotPixelThreshold] 

344 valueThreshold = stDev*hotPixelThreshold 

345 elif datasetType.lower() == 'bias': 

346 self.log.warning( 

347 "Bias frame detected, but thresholdType == STDEV; not looking for defects.", 

348 ) 

349 return Defects.fromFootprintList([]) 

350 else: 

351 nSigmaList = [hotPixelThreshold, coldPixelThreshold] 

352 valueThreshold = [x*stDev for x in nSigmaList] 

353 

354 self.log.info("Image type: %s. Amp: %s. Threshold Type: %s. Sigma values and Pixel" 

355 "Values (hot and cold pixels thresholds): %s, %s", 

356 datasetType, ampName, thresholdType, nSigmaList, valueThreshold) 

357 mergedSet = None 

358 for sigma in nSigmaList: 

359 nSig = np.abs(sigma) 

360 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

361 polarity = {-1: False, 1: True}[np.sign(sigma)] 

362 

363 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

364 

365 try: 

366 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

367 except InvalidParameterError: 

368 # This occurs if the image sigma value is 0.0. 

369 # Let's mask the whole area. 

370 minValue = np.nanmin(ampImg.image.array) - 1.0 

371 threshold = afwDetection.createThreshold(minValue, 'value', polarity=True) 

372 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

373 

374 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

375 

376 if mergedSet is None: 

377 mergedSet = footprintSet 

378 else: 

379 mergedSet.merge(footprintSet) 

380 

381 if polarity: 

382 # hot pixels 

383 for fp in footprintSet.getFootprints(): 

384 hotPixelCount[ampName] += fp.getArea() 

385 else: 

386 # cold pixels 

387 for fp in footprintSet.getFootprints(): 

388 coldPixelCount[ampName] += fp.getArea() 

389 

390 footprintList += mergedSet.getFootprints() 

391 

392 self.debugView('defectMap', ampImg, 

393 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

394 

395 defects = Defects.fromFootprintList(footprintList) 

396 defects = self.dilateSaturatedColumns(exp, defects) 

397 defects, _ = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

398 defects, count = self.maskBadColumns(exp, defects) 

399 # We want this to reflect the number of completely bad columns. 

400 defects.updateCounters(columns=count, hot=hotPixelCount, cold=coldPixelCount) 

401 

402 return defects 

403 

404 @staticmethod 

405 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

406 """Return the number of non-bad pixels in the image.""" 

407 nPixels = maskedIm.mask.array.size 

408 nBad = countMaskedPixels(maskedIm, badMaskString) 

409 return nPixels - nBad 

410 

411 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

412 """Set edge bits on an exposure or maskedImage. 

413 

414 Raises 

415 ------ 

416 TypeError 

417 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

418 """ 

419 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

420 mi = exposureOrMaskedImage.maskedImage 

421 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

422 mi = exposureOrMaskedImage 

423 else: 

424 t = type(exposureOrMaskedImage) 

425 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

426 

427 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

428 if self.config.nPixBorderLeftRight: 

429 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

430 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

431 if self.config.nPixBorderUpDown: 

432 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

433 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

434 

435 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

436 """Mask blocks in a column if there are on-and-off bad pixels 

437 

438 If there's a column with on and off bad pixels, mask all the 

439 pixels in between, except if there is a large enough gap of 

440 consecutive good pixels between two bad pixels in the column. 

441 

442 Parameters 

443 ---------- 

444 defects : `lsst.ip.isr.Defects` 

445 The defects found in the image so far 

446 

447 Returns 

448 ------- 

449 defects : `lsst.ip.isr.Defects` 

450 If the number of bad pixels in a column is not larger or 

451 equal than self.config.badPixelColumnThreshold, the input 

452 list is returned. Otherwise, the defects list returned 

453 will include boxes that mask blocks of on-and-of pixels. 

454 badColumnCount : `int` 

455 Number of bad columns partially masked. 

456 """ 

457 badColumnCount = 0 

458 # Get the (x, y) values of each bad pixel in amp. 

459 coordinates = [] 

460 for defect in defects: 

461 bbox = defect.getBBox() 

462 x0, y0 = bbox.getMinX(), bbox.getMinY() 

463 deltaX0, deltaY0 = bbox.getDimensions() 

464 for j in np.arange(y0, y0+deltaY0): 

465 for i in np.arange(x0, x0 + deltaX0): 

466 coordinates.append((i, j)) 

467 

468 x, y = [], [] 

469 for coordinatePair in coordinates: 

470 x.append(coordinatePair[0]) 

471 y.append(coordinatePair[1]) 

472 

473 x = np.array(x) 

474 y = np.array(y) 

475 # Find the defects with same "x" (vertical) coordinate (column). 

476 unique, counts = np.unique(x, return_counts=True) 

477 multipleX = [] 

478 for (a, b) in zip(unique, counts): 

479 if b >= self.config.badOnAndOffPixelColumnThreshold: 

480 multipleX.append(a) 

481 if len(multipleX) != 0: 

482 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

483 badColumnCount += 1 

484 

485 return defects, badColumnCount 

486 

487 def dilateSaturatedColumns(self, exp, defects): 

488 """Dilate saturated columns by a configurable amount. 

489 

490 Parameters 

491 ---------- 

492 exp : `lsst.afw.image.exposure.Exposure` 

493 The exposure in which to find defects. 

494 defects : `lsst.ip.isr.Defects` 

495 The defects found in the image so far 

496 

497 Returns 

498 ------- 

499 defects : `lsst.ip.isr.Defects` 

500 The expanded defects. 

501 """ 

502 if self.config.saturatedColumnDilationRadius <= 0: 

503 # This is a no-op. 

504 return defects 

505 

506 mask = afwImage.Mask.getPlaneBitMask(self.config.saturatedColumnMask) 

507 

508 satY, satX = np.where((exp.mask.array & mask) > 0) 

509 

510 if len(satX) == 0: 

511 # No saturated pixels, nothing to do. 

512 return defects 

513 

514 radius = self.config.saturatedColumnDilationRadius 

515 

516 with defects.bulk_update(): 

517 for index in range(len(satX)): 

518 minX = np.clip(satX[index] - radius, 0, None) 

519 maxX = np.clip(satX[index] + radius, None, exp.image.array.shape[1] - 1) 

520 s = Box2I(minimum=Point2I(minX, satY[index]), 

521 maximum=Point2I(maxX, satY[index])) 

522 defects.append(s) 

523 

524 return defects 

525 

526 def maskBadColumns(self, exp, defects): 

527 """Mask full amplifier columns if they are sufficiently bad. 

528 

529 Parameters 

530 ---------- 

531 defects : `lsst.ip.isr.Defects` 

532 The defects found in the image so far 

533 

534 Returns 

535 ------- 

536 exp : `lsst.afw.image.exposure.Exposure` 

537 The exposure in which to find defects. 

538 defects : `lsst.ip.isr.Defects` 

539 If the number of bad pixels in a column is not larger or 

540 equal than self.config.badPixelColumnThreshold, the input 

541 list is returned. Otherwise, the defects list returned 

542 will include boxes that mask blocks of on-and-of pixels. 

543 badColumnCount : `int` 

544 Number of bad columns masked. 

545 """ 

546 # Render the defects into an image. 

547 defectImage = afwImage.ImageI(exp.getBBox()) 

548 

549 for defect in defects: 

550 defectImage[defect.getBBox()] = 1 

551 

552 badColumnCount = 0 

553 

554 if self.config.badPixelsToFillColumnThreshold > 0: 

555 with defects.bulk_update(): 

556 for amp in exp.getDetector(): 

557 subImage = defectImage[amp.getBBox()].array 

558 nInCol = np.sum(subImage, axis=0) 

559 

560 badColIndices, = (nInCol >= self.config.badPixelsToFillColumnThreshold).nonzero() 

561 badColumns = badColIndices + amp.getBBox().getMinX() 

562 

563 for badColumn in badColumns: 

564 s = Box2I(minimum=Point2I(badColumn, amp.getBBox().getMinY()), 

565 maximum=Point2I(badColumn, amp.getBBox().getMaxY())) 

566 defects.append(s) 

567 

568 badColumnCount += len(badColIndices) 

569 

570 if self.config.saturatedPixelsToFillColumnThreshold > 0: 

571 mask = afwImage.Mask.getPlaneBitMask(self.config.saturatedColumnMask) 

572 

573 with defects.bulk_update(): 

574 for amp in exp.getDetector(): 

575 subMask = exp.mask[amp.getBBox()].array 

576 # Turn all the SAT bits into 1s 

577 subMask &= mask 

578 subMask[subMask > 0] = 1 

579 

580 nInCol = np.sum(subMask, axis=0) 

581 

582 badColIndices, = (nInCol >= self.config.saturatedPixelsToFillColumnThreshold).nonzero() 

583 badColumns = badColIndices + amp.getBBox().getMinX() 

584 

585 for badColumn in badColumns: 

586 s = Box2I(minimum=Point2I(badColumn, amp.getBBox().getMinY()), 

587 maximum=Point2I(badColumn, amp.getBBox().getMaxY())) 

588 defects.append(s) 

589 

590 badColumnCount += len(badColIndices) 

591 

592 return defects, badColumnCount 

593 

594 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

595 """Mask blocks in a column if number of on-and-off bad pixels is above 

596 threshold. 

597 

598 This function is called if the number of on-and-off bad pixels 

599 in a column is larger or equal than 

600 self.config.badOnAndOffPixelColumnThreshold. 

601 

602 Parameters 

603 --------- 

604 x : `list` 

605 Lower left x coordinate of defect box. x coordinate is 

606 along the short axis if amp. 

607 y : `list` 

608 Lower left y coordinate of defect box. x coordinate is 

609 along the long axis if amp. 

610 multipleX : list 

611 List of x coordinates in amp. with multiple bad pixels 

612 (i.e., columns with defects). 

613 defects : `lsst.ip.isr.Defects` 

614 The defcts found in the image so far 

615 

616 Returns 

617 ------- 

618 defects : `lsst.ip.isr.Defects` 

619 The defects list returned that will include boxes that 

620 mask blocks of on-and-of pixels. 

621 """ 

622 with defects.bulk_update(): 

623 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

624 for x0 in multipleX: 

625 index = np.where(x == x0) 

626 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

627 multipleY.sort() # Ensure that the y values are sorted to look for gaps. 

628 minY, maxY = np.min(multipleY), np.max(multipleY) 

629 # Next few lines: don't mask pixels in column if gap 

630 # of good pixels between two consecutive bad pixels is 

631 # larger or equal than 'goodPixelColumnGapThreshold'. 

632 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

633 if len(diffIndex) != 0: 

634 limits = [minY] # put the minimum first 

635 for gapIndex in diffIndex: 

636 limits.append(multipleY[gapIndex]) 

637 limits.append(multipleY[gapIndex+1]) 

638 limits.append(maxY) # maximum last 

639 for i in np.arange(0, len(limits)-1, 2): 

640 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

641 defects.append(s) 

642 else: # No gap is large enough 

643 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

644 defects.append(s) 

645 return defects 

646 

647 def debugView(self, stepname, ampImage, defects, detector): # pragma: no cover 

648 """Plot the defects found by the task. 

649 

650 Parameters 

651 ---------- 

652 stepname : `str` 

653 Debug frame to request. 

654 ampImage : `lsst.afw.image.MaskedImage` 

655 Amplifier image to display. 

656 defects : `lsst.ip.isr.Defects` 

657 The defects to plot. 

658 detector : `lsst.afw.cameraGeom.Detector` 

659 Detector holding camera geometry. 

660 """ 

661 frame = getDebugFrame(self._display, stepname) 

662 if frame: 

663 disp = afwDisplay.Display(frame=frame) 

664 disp.scale('asinh', 'zscale') 

665 disp.setMaskTransparency(80) 

666 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

667 

668 maskedIm = ampImage.clone() 

669 defects.maskPixels(maskedIm, "BAD") 

670 

671 mpDict = maskedIm.mask.getMaskPlaneDict() 

672 for plane in mpDict.keys(): 

673 if plane in ['BAD']: 

674 continue 

675 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

676 

677 disp.setImageColormap('gray') 

678 disp.mtv(maskedIm) 

679 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

680 prompt = "Press Enter to continue [c]... " 

681 while True: 

682 ans = input(prompt).lower() 

683 if ans in ('', 'c', ): 

684 break 

685 

686 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

687 """Make a histogram of the distribution of pixel values for 

688 each amp. 

689 

690 The main image data histogram is plotted in blue. Edge 

691 pixels, if masked, are in red. Note that masked edge pixels 

692 do not contribute to the underflow and overflow numbers. 

693 

694 Note that this currently only supports the 16-amp LSST 

695 detectors. 

696 

697 Parameters 

698 ---------- 

699 stepname : `str` 

700 Debug frame to request. 

701 ampImage : `lsst.afw.image.MaskedImage` 

702 Amplifier image to display. 

703 nSigmaUsed : `float` 

704 The number of sigma used for detection 

705 exp : `lsst.afw.image.exposure.Exposure` 

706 The exposure in which the defects were found. 

707 """ 

708 frame = getDebugFrame(self._display, stepname) 

709 if frame: 

710 import matplotlib.pyplot as plt 

711 

712 detector = exp.getDetector() 

713 nX = np.floor(np.sqrt(len(detector))) 

714 nY = len(detector) // nX 

715 fig, ax = plt.subplots(nrows=int(nY), ncols=int(nX), sharex='col', sharey='row', figsize=(13, 10)) 

716 

717 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

718 

719 for (amp, a) in zip(reversed(detector), ax.flatten()): 

720 mi = exp.maskedImage[amp.getBBox()] 

721 

722 # normalize by expTime as we plot in ADU/s and don't 

723 # always work with master calibs 

724 mi.image.array /= expTime 

725 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

726 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

727 # Get array of pixels 

728 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

729 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

730 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

731 

732 thrUpper = mean + nSigmaUsed*sigma 

733 thrLower = mean - nSigmaUsed*sigma 

734 

735 nRight = len(imgData[imgData > thrUpper]) 

736 nLeft = len(imgData[imgData < thrLower]) 

737 

738 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

739 leftEdge = mean - nsig * nSigmaUsed*sigma 

740 rightEdge = mean + nsig * nSigmaUsed*sigma 

741 nbins = np.linspace(leftEdge, rightEdge, 1000) 

742 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

743 lw=1, edgecolor='red') 

744 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

745 lw=3, edgecolor='blue') 

746 

747 # Report number of entries in over- and under-flow 

748 # bins, i.e. off the edges of the histogram 

749 nOverflow = len(imgData[imgData > rightEdge]) 

750 nUnderflow = len(imgData[imgData < leftEdge]) 

751 

752 # Put v-lines and textboxes in 

753 a.axvline(thrUpper, c='k') 

754 a.axvline(thrLower, c='k') 

755 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

756 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

757 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

758 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

759 

760 # set axis limits and scales 

761 a.set_ylim([1., 1.7*np.max(y)]) 

762 lPlot, rPlot = a.get_xlim() 

763 a.set_xlim(np.array([lPlot, rPlot])) 

764 a.set_yscale('log') 

765 a.set_xlabel("ADU/s") 

766 fig.show() 

767 prompt = "Press Enter or c to continue [chp]..." 

768 while True: 

769 ans = input(prompt).lower() 

770 if ans in ("", " ", "c",): 

771 break 

772 elif ans in ("p", ): 

773 import pdb 

774 pdb.set_trace() 

775 elif ans in ("h", ): 

776 print("[h]elp [c]ontinue [p]db") 

777 plt.close() 

778 

779 

780class MeasureDefectsCombinedConnections(pipeBase.PipelineTaskConnections, 

781 dimensions=("instrument", "detector")): 

782 inputExp = cT.Input( 

783 name="dark", 

784 doc="Input ISR-processed combined exposure to measure.", 

785 storageClass="ExposureF", 

786 dimensions=("instrument", "detector"), 

787 multiple=False, 

788 isCalibration=True, 

789 ) 

790 camera = cT.PrerequisiteInput( 

791 name='camera', 

792 doc="Camera associated with this exposure.", 

793 storageClass="Camera", 

794 dimensions=("instrument", ), 

795 isCalibration=True, 

796 ) 

797 

798 outputDefects = cT.Output( 

799 name="cpPartialDefectsFromDarkCombined", 

800 doc="Output measured defects.", 

801 storageClass="Defects", 

802 dimensions=("instrument", "detector"), 

803 ) 

804 

805 

806class MeasureDefectsCombinedTaskConfig(MeasureDefectsTaskConfig, 

807 pipelineConnections=MeasureDefectsCombinedConnections): 

808 """Configuration for measuring defects from combined exposures. 

809 """ 

810 pass 

811 

812 

813class MeasureDefectsCombinedTask(MeasureDefectsTask): 

814 """Task to measure defects in combined images.""" 

815 

816 ConfigClass = MeasureDefectsCombinedTaskConfig 

817 _DefaultName = "cpDefectMeasureCombined" 

818 

819 

820class MeasureDefectsCombinedWithFilterConnections(pipeBase.PipelineTaskConnections, 

821 dimensions=("instrument", "detector", "physical_filter")): 

822 """Task to measure defects in combined flats under a certain filter.""" 

823 inputExp = cT.Input( 

824 name="flat", 

825 doc="Input ISR-processed combined exposure to measure.", 

826 storageClass="ExposureF", 

827 dimensions=("instrument", "detector", "physical_filter"), 

828 multiple=False, 

829 isCalibration=True, 

830 ) 

831 camera = cT.PrerequisiteInput( 

832 name='camera', 

833 doc="Camera associated with this exposure.", 

834 storageClass="Camera", 

835 dimensions=("instrument", ), 

836 isCalibration=True, 

837 ) 

838 

839 outputDefects = cT.Output( 

840 name="cpPartialDefectsFromFlatCombinedWithFilter", 

841 doc="Output measured defects.", 

842 storageClass="Defects", 

843 dimensions=("instrument", "detector", "physical_filter"), 

844 ) 

845 

846 

847class MeasureDefectsCombinedWithFilterTaskConfig( 

848 MeasureDefectsTaskConfig, 

849 pipelineConnections=MeasureDefectsCombinedWithFilterConnections): 

850 """Configuration for measuring defects from combined exposures. 

851 """ 

852 pass 

853 

854 

855class MeasureDefectsCombinedWithFilterTask(MeasureDefectsTask): 

856 """Task to measure defects in combined images.""" 

857 

858 ConfigClass = MeasureDefectsCombinedWithFilterTaskConfig 

859 _DefaultName = "cpDefectMeasureWithFilterCombined" 

860 

861 

862class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

863 dimensions=("instrument", "detector")): 

864 inputDefects = cT.Input( 

865 name="singleExpDefects", 

866 doc="Measured defect lists.", 

867 storageClass="Defects", 

868 dimensions=("instrument", "detector", "exposure",), 

869 multiple=True, 

870 ) 

871 camera = cT.PrerequisiteInput( 

872 name='camera', 

873 doc="Camera associated with these defects.", 

874 storageClass="Camera", 

875 dimensions=("instrument", ), 

876 isCalibration=True, 

877 ) 

878 

879 mergedDefects = cT.Output( 

880 name="defects", 

881 doc="Final merged defects.", 

882 storageClass="Defects", 

883 dimensions=("instrument", "detector"), 

884 multiple=False, 

885 isCalibration=True, 

886 ) 

887 

888 

889class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

890 pipelineConnections=MergeDefectsConnections): 

891 """Configuration for merging single exposure defects. 

892 """ 

893 

894 assertSameRun = pexConfig.Field( 

895 dtype=bool, 

896 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or " 

897 "if the run key isn't found."), 

898 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

899 ) 

900 ignoreFilters = pexConfig.Field( 

901 dtype=bool, 

902 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

903 " images. Allows mixing of filters in the input flats. Set to False if you think" 

904 " your defects might be chromatic and want to have registry support for varying" 

905 " defects with respect to filter."), 

906 default=True, 

907 ) 

908 nullFilterName = pexConfig.Field( 

909 dtype=str, 

910 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

911 default="NONE", 

912 ) 

913 combinationMode = pexConfig.ChoiceField( 

914 doc="Which types of defects to identify", 

915 dtype=str, 

916 default="FRACTION", 

917 allowed={ 

918 "AND": "Logical AND the pixels found in each visit to form set ", 

919 "OR": "Logical OR the pixels found in each visit to form set ", 

920 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

921 } 

922 ) 

923 combinationFraction = pexConfig.RangeField( 

924 dtype=float, 

925 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

926 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

927 " mode AND to require pixel to appear in all images."), 

928 default=0.7, 

929 min=0, 

930 max=1, 

931 ) 

932 nPixBorderUpDown = pexConfig.Field( 

933 dtype=int, 

934 doc="Number of pixels on top & bottom of image to mask as defects if edgesAsDefects is True.", 

935 default=5, 

936 ) 

937 nPixBorderLeftRight = pexConfig.Field( 

938 dtype=int, 

939 doc="Number of pixels on left & right of image to mask as defects if edgesAsDefects is True.", 

940 default=5, 

941 ) 

942 edgesAsDefects = pexConfig.Field( 

943 dtype=bool, 

944 doc="Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects.", 

945 default=False, 

946 ) 

947 

948 

949class MergeDefectsTask(pipeBase.PipelineTask): 

950 """Merge the defects from multiple exposures. 

951 """ 

952 

953 ConfigClass = MergeDefectsTaskConfig 

954 _DefaultName = 'cpDefectMerge' 

955 

956 def run(self, inputDefects, camera): 

957 """Merge a list of single defects to find the common defect regions. 

958 

959 Parameters 

960 ---------- 

961 inputDefects : `list` [`lsst.ip.isr.Defects`] 

962 Partial defects from a single exposure. 

963 camera : `lsst.afw.cameraGeom.Camera` 

964 Camera to use for metadata. 

965 

966 Returns 

967 ------- 

968 results : `lsst.pipe.base.Struct` 

969 Results struct containing: 

970 

971 ``mergedDefects`` 

972 The defects merged from the input lists 

973 (`lsst.ip.isr.Defects`). 

974 """ 

975 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

976 if detectorId is None: 

977 raise RuntimeError("Cannot identify detector id.") 

978 detector = camera[detectorId] 

979 

980 imageTypes = set() 

981 for inDefect in inputDefects: 

982 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

983 imageTypes.add(imageType) 

984 

985 # Determine common defect pixels separately for each input image type. 

986 splitDefects = list() 

987 for imageType in imageTypes: 

988 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

989 count = 0 

990 for inDefect in inputDefects: 

991 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

992 count += 1 

993 for defect in inDefect: 

994 sumImage.image[defect.getBBox()] += 1.0 

995 sumImage /= count 

996 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

997 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

998 

999 if self.config.combinationMode == 'AND': 

1000 threshold = 1.0 

1001 elif self.config.combinationMode == 'OR': 

1002 threshold = 0.0 

1003 elif self.config.combinationMode == 'FRACTION': 

1004 threshold = self.config.combinationFraction 

1005 else: 

1006 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

1007 indices = np.where(sumImage.getImage().getArray() > threshold) 

1008 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

1009 sumImage.getMask().getArray()[indices] |= BADBIT 

1010 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

1011 partialDefect = Defects.fromMask(sumImage, 'BAD') 

1012 splitDefects.append(partialDefect) 

1013 

1014 # Do final combination of separate image types 

1015 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

1016 for inDefect in splitDefects: 

1017 for defect in inDefect: 

1018 finalImage.image[defect.getBBox()] += 1 

1019 finalImage /= len(splitDefects) 

1020 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) 

1021 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) 

1022 

1023 # This combination is the OR of all image types 

1024 threshold = 0.0 

1025 indices = np.where(finalImage.getImage().getArray() > threshold) 

1026 BADBIT = finalImage.getMask().getPlaneBitMask('BAD') 

1027 finalImage.getMask().getArray()[indices] |= BADBIT 

1028 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) 

1029 

1030 if self.config.edgesAsDefects: 

1031 self.log.info("Masking edge pixels as defects.") 

1032 # This code follows the pattern from isrTask.maskEdges(). 

1033 if self.config.nPixBorderLeftRight > 0: 

1034 box = detector.getBBox() 

1035 subImage = finalImage[box] 

1036 box.grow(Extent2I(-self.config.nPixBorderLeftRight, 0)) 

1037 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

1038 if self.config.nPixBorderUpDown > 0: 

1039 box = detector.getBBox() 

1040 subImage = finalImage[box] 

1041 box.grow(Extent2I(0, -self.config.nPixBorderUpDown)) 

1042 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

1043 

1044 merged = Defects.fromMask(finalImage, 'BAD') 

1045 merged.updateMetadataFromExposures(inputDefects) 

1046 merged.updateMetadata(camera=camera, detector=detector, filterName=None, 

1047 setCalibId=True, setDate=True) 

1048 

1049 return pipeBase.Struct( 

1050 mergedDefects=merged, 

1051 ) 

1052 

1053# Subclass the MergeDefects task to reduce the input dimensions 

1054# from ("instrument", "detector", "exposure") to 

1055# ("instrument", "detector"). 

1056 

1057 

1058class MergeDefectsCombinedConnections(pipeBase.PipelineTaskConnections, 

1059 dimensions=("instrument", "detector")): 

1060 inputDarkDefects = cT.Input( 

1061 name="cpPartialDefectsFromDarkCombined", 

1062 doc="Measured defect lists.", 

1063 storageClass="Defects", 

1064 dimensions=("instrument", "detector",), 

1065 multiple=True, 

1066 ) 

1067 inputBiasDefects = cT.Input( 

1068 name="cpPartialDefectsFromBiasCombined", 

1069 doc="Additional measured defect lists.", 

1070 storageClass="Defects", 

1071 dimensions=("instrument", "detector",), 

1072 multiple=True, 

1073 ) 

1074 inputFlatDefects = cT.Input( 

1075 name="cpPartialDefectsFromFlatCombinedWithFilter", 

1076 doc="Additional measured defect lists.", 

1077 storageClass="Defects", 

1078 dimensions=("instrument", "detector", "physical_filter"), 

1079 multiple=True, 

1080 ) 

1081 camera = cT.PrerequisiteInput( 

1082 name='camera', 

1083 doc="Camera associated with these defects.", 

1084 storageClass="Camera", 

1085 dimensions=("instrument", ), 

1086 isCalibration=True, 

1087 ) 

1088 

1089 mergedDefects = cT.Output( 

1090 name="defects", 

1091 doc="Final merged defects.", 

1092 storageClass="Defects", 

1093 dimensions=("instrument", "detector"), 

1094 multiple=False, 

1095 isCalibration=True, 

1096 ) 

1097 

1098 

1099class MergeDefectsCombinedTaskConfig(MergeDefectsTaskConfig, 

1100 pipelineConnections=MergeDefectsCombinedConnections): 

1101 """Configuration for merging defects from combined exposure. 

1102 """ 

1103 def validate(self): 

1104 super().validate() 

1105 if self.combinationMode != 'OR': 

1106 raise ValueError("combinationMode must be 'OR'") 

1107 

1108 

1109class MergeDefectsCombinedTask(MergeDefectsTask): 

1110 """Task to measure defects in combined images.""" 

1111 

1112 ConfigClass = MergeDefectsCombinedTaskConfig 

1113 _DefaultName = "cpDefectMergeCombined" 

1114 

1115 @staticmethod 

1116 def chooseBest(inputs): 

1117 """Select the input with the most exposures used.""" 

1118 best = 0 

1119 if len(inputs) > 1: 

1120 nInput = 0 

1121 for num, exp in enumerate(inputs): 

1122 # This technically overcounts by a factor of 3. 

1123 N = len([k for k, v in exp.getMetadata().toDict().items() if "CPP_INPUT_" in k]) 

1124 if N > nInput: 

1125 best = num 

1126 nInput = N 

1127 return inputs[best] 

1128 

1129 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1130 inputs = butlerQC.get(inputRefs) 

1131 # Turn inputFlatDefects and inputDarkDefects into a list which 

1132 # is what MergeDefectsTask expects. If there are multiple, 

1133 # use the one with the most inputs. 

1134 tempList = [self.chooseBest(inputs['inputFlatDefects']), 

1135 self.chooseBest(inputs['inputDarkDefects']), 

1136 self.chooseBest(inputs['inputBiasDefects'])] 

1137 

1138 # Rename inputDefects 

1139 inputsCombined = {'inputDefects': tempList, 'camera': inputs['camera']} 

1140 

1141 outputs = super().run(**inputsCombined) 

1142 butlerQC.put(outputs, outputRefs)