Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.pipe.base as pipeBase 

25import lsst.pipe.base.connectionTypes as cT 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.detection as afwDetection 

33import lsst.afw.display as afwDisplay 

34from lsst.afw import cameraGeom 

35from lsst.geom import Box2I, Point2I 

36from lsst.meas.algorithms import SourceDetectionTask 

37from lsst.ip.isr import IsrTask, Defects 

38from .utils import countMaskedPixels 

39from lsst.pipe.tasks.getRepositoryData import DataRefListRunner 

40 

41from ._lookupStaticCalibration import lookupStaticCalibration 

42 

43__all__ = ['MeasureDefectsTaskConfig', 'MeasureDefectsTask', 

44 'MergeDefectsTaskConfig', 'MergeDefectsTask', 

45 'FindDefectsTask', 'FindDefectsTaskConfig', ] 

46 

47 

48class MeasureDefectsConnections(pipeBase.PipelineTaskConnections, 

49 dimensions=("instrument", "exposure", "detector")): 

50 inputExp = cT.Input( 

51 name="defectExps", 

52 doc="Input ISR-processed exposures to measure.", 

53 storageClass="Exposure", 

54 dimensions=("instrument", "detector", "exposure"), 

55 multiple=False 

56 ) 

57 camera = cT.PrerequisiteInput( 

58 name='camera', 

59 doc="Camera associated with this exposure.", 

60 storageClass="Camera", 

61 dimensions=("instrument", ), 

62 isCalibration=True, 

63 lookupFunction=lookupStaticCalibration, 

64 ) 

65 

66 outputDefects = cT.Output( 

67 name="singleExpDefects", 

68 doc="Output measured defects.", 

69 storageClass="Defects", 

70 dimensions=("instrument", "detector", "exposure"), 

71 ) 

72 

73 

74class MeasureDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

75 pipelineConnections=MeasureDefectsConnections): 

76 """Configuration for measuring defects from a list of exposures 

77 """ 

78 nSigmaBright = pexConfig.Field( 

79 dtype=float, 

80 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be", 

81 " appropriate for some LSST sensors in DM-17490."), 

82 default=4.8, 

83 ) 

84 nSigmaDark = pexConfig.Field( 

85 dtype=float, 

86 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be", 

87 " appropriate for some LSST sensors in DM-17490."), 

88 default=-5.0, 

89 ) 

90 nPixBorderUpDown = pexConfig.Field( 

91 dtype=int, 

92 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

93 default=7, 

94 ) 

95 nPixBorderLeftRight = pexConfig.Field( 

96 dtype=int, 

97 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

98 default=7, 

99 ) 

100 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

101 dtype=int, 

102 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ", 

103 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ", 

104 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ", 

105 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ", 

106 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ", 

107 "non-bad pixels in BPC, an exception to the above is made and those consecutive ", 

108 "'goodPixelColumnGapThreshold' are not marked as bad."), 

109 default=50, 

110 ) 

111 goodPixelColumnGapThreshold = pexConfig.Field( 

112 dtype=int, 

113 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ", 

114 "'badOnAndOffPixelColumnThreshold')."), 

115 default=30, 

116 ) 

117 

118 def validate(self): 

119 super().validate() 

120 if self.nSigmaBright < 0.0: 

121 raise ValueError("nSigmaBright must be above 0.0.") 

122 if self.nSigmaDark > 0.0: 

123 raise ValueError("nSigmaDark must be below 0.0.") 

124 

125 

126class MeasureDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

127 """Measure the defects from one exposure. 

128 """ 

129 ConfigClass = MeasureDefectsTaskConfig 

130 _DefaultName = 'cpDefectMeasure' 

131 

132 def run(self, inputExp, camera): 

133 """Measure one exposure for defects. 

134 

135 Parameters 

136 ---------- 

137 inputExp : `lsst.afw.image.Exposure` 

138 Exposure to examine. 

139 camera : `lsst.afw.cameraGeom.Camera` 

140 Camera to use for metadata. 

141 

142 Returns 

143 ------- 

144 results : `lsst.pipe.base.Struct` 

145 Results struct containing: 

146 - ``outputDefects` : `lsst.ip.isr.Defects` 

147 The defects measured from this exposure. 

148 """ 

149 detector = inputExp.getDetector() 

150 

151 filterName = inputExp.getFilter().getName() 

152 datasetType = inputExp.getMetadata().get('IMGTYPE', 'UNKNOWN') 

153 

154 if datasetType.lower() == 'dark': 

155 nSigmaList = [self.config.nSigmaBright] 

156 else: 

157 nSigmaList = [self.config.nSigmaBright, self.config.nSigmaDark] 

158 defects = self.findHotAndColdPixels(inputExp, nSigmaList) 

159 

160 msg = "Found %s defects containing %s pixels in %s" 

161 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

162 

163 defects.updateMetadata(camera=camera, detector=detector, filterName=filterName, 

164 setCalibId=True, setDate=True, 

165 cpDefectGenImageType=datasetType) 

166 

167 return pipeBase.Struct( 

168 outputDefects=defects, 

169 ) 

170 

171 @staticmethod 

172 def _nPixFromDefects(defects): 

173 """Count pixels in a defect. 

174 

175 Parameters 

176 ---------- 

177 defects : `lsst.ip.isr.Defects` 

178 Defects to measure. 

179 

180 Returns 

181 ------- 

182 nPix : `int` 

183 Number of defect pixels. 

184 """ 

185 nPix = 0 

186 for defect in defects: 

187 nPix += defect.getBBox().getArea() 

188 return nPix 

189 

190 def findHotAndColdPixels(self, exp, nSigma): 

191 """Find hot and cold pixels in an image. 

192 

193 Using config-defined thresholds on a per-amp basis, mask 

194 pixels that are nSigma above threshold in dark frames (hot 

195 pixels), or nSigma away from the clipped mean in flats (hot & 

196 cold pixels). 

197 

198 Parameters 

199 ---------- 

200 exp : `lsst.afw.image.exposure.Exposure` 

201 The exposure in which to find defects. 

202 nSigma : `list [ `float` ] 

203 Detection threshold to use. Positive for DETECTED pixels, 

204 negative for DETECTED_NEGATIVE pixels. 

205 

206 Returns 

207 ------- 

208 defects : `lsst.ip.isr.Defect` 

209 The defects found in the image. 

210 

211 """ 

212 

213 self._setEdgeBits(exp) 

214 maskedIm = exp.maskedImage 

215 

216 # the detection polarity for afwDetection, True for positive, 

217 # False for negative, and therefore True for darks as they only have 

218 # bright pixels, and both for flats, as they have bright and dark pix 

219 footprintList = [] 

220 

221 for amp in exp.getDetector(): 

222 ampImg = maskedIm[amp.getBBox()].clone() 

223 

224 # crop ampImage depending on where the amp lies in the image 

225 if self.config.nPixBorderLeftRight: 

226 if ampImg.getX0() == 0: 

227 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

228 else: 

229 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

230 if self.config.nPixBorderUpDown: 

231 if ampImg.getY0() == 0: 

232 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

233 else: 

234 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

235 

236 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

237 continue 

238 

239 # Remove a background estimate 

240 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

241 

242 mergedSet = None 

243 for sigma in nSigma: 

244 nSig = np.abs(sigma) 

245 self.debugHistogram('ampFlux', ampImg, nSig, exp) 

246 polarity = {-1: False, 1: True}[np.sign(sigma)] 

247 

248 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

249 

250 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

251 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

252 

253 if mergedSet is None: 

254 mergedSet = footprintSet 

255 else: 

256 mergedSet.merge(footprintSet) 

257 

258 footprintList += mergedSet.getFootprints() 

259 

260 self.debugView('defectMap', ampImg, 

261 Defects.fromFootprintList(mergedSet.getFootprints()), exp.getDetector()) 

262 

263 defects = Defects.fromFootprintList(footprintList) 

264 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

265 

266 return defects 

267 

268 @staticmethod 

269 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

270 """Return the number of non-bad pixels in the image.""" 

271 nPixels = maskedIm.mask.array.size 

272 nBad = countMaskedPixels(maskedIm, badMaskString) 

273 return nPixels - nBad 

274 

275 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

276 """Set edge bits on an exposure or maskedImage. 

277 Raises 

278 ------ 

279 TypeError 

280 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

281 """ 

282 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

283 mi = exposureOrMaskedImage.maskedImage 

284 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

285 mi = exposureOrMaskedImage 

286 else: 

287 t = type(exposureOrMaskedImage) 

288 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

289 

290 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

291 if self.config.nPixBorderLeftRight: 

292 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

293 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

294 if self.config.nPixBorderUpDown: 

295 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

296 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

297 

298 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

299 """Mask blocks in a column if there are on-and-off bad pixels 

300 

301 If there's a column with on and off bad pixels, mask all the 

302 pixels in between, except if there is a large enough gap of 

303 consecutive good pixels between two bad pixels in the column. 

304 

305 Parameters 

306 --------- 

307 defects: `lsst.ip.isr.Defect` 

308 The defects found in the image so far 

309 

310 Returns 

311 ------ 

312 defects: `lsst.ip.isr.Defect` 

313 If the number of bad pixels in a column is not larger or 

314 equal than self.config.badPixelColumnThreshold, the iput 

315 list is returned. Otherwise, the defects list returned 

316 will include boxes that mask blocks of on-and-of pixels. 

317 

318 """ 

319 # Get the (x, y) values of each bad pixel in amp. 

320 coordinates = [] 

321 for defect in defects: 

322 bbox = defect.getBBox() 

323 x0, y0 = bbox.getMinX(), bbox.getMinY() 

324 deltaX0, deltaY0 = bbox.getDimensions() 

325 for j in np.arange(y0, y0+deltaY0): 

326 for i in np.arange(x0, x0 + deltaX0): 

327 coordinates.append((i, j)) 

328 

329 x, y = [], [] 

330 for coordinatePair in coordinates: 

331 x.append(coordinatePair[0]) 

332 y.append(coordinatePair[1]) 

333 

334 x = np.array(x) 

335 y = np.array(y) 

336 # Find the defects with same "x" (vertical) coordinate (column). 

337 unique, counts = np.unique(x, return_counts=True) 

338 multipleX = [] 

339 for (a, b) in zip(unique, counts): 

340 if b >= self.config.badOnAndOffPixelColumnThreshold: 

341 multipleX.append(a) 

342 if len(multipleX) != 0: 

343 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

344 

345 return defects 

346 

347 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

348 """Mask blocks in a column if number of on-and-off bad pixels is above threshold. 

349 

350 This function is called if the number of on-and-off bad pixels 

351 in a column is larger or equal than 

352 self.config.badOnAndOffPixelColumnThreshold. 

353 

354 Parameters 

355 --------- 

356 x: `list` 

357 Lower left x coordinate of defect box. x coordinate is 

358 along the short axis if amp. 

359 y: `list` 

360 Lower left y coordinate of defect box. x coordinate is 

361 along the long axis if amp. 

362 multipleX: list 

363 List of x coordinates in amp. with multiple bad pixels 

364 (i.e., columns with defects). 

365 defects: `lsst.ip.isr.Defect` 

366 The defcts found in the image so far 

367 

368 Returns 

369 ------- 

370 defects: `lsst.ip.isr.Defect` 

371 The defects list returned that will include boxes that 

372 mask blocks of on-and-of pixels. 

373 

374 """ 

375 with defects.bulk_update(): 

376 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

377 for x0 in multipleX: 

378 index = np.where(x == x0) 

379 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

380 minY, maxY = np.min(multipleY), np.max(multipleY) 

381 # Next few lines: don't mask pixels in column if gap of good pixels between 

382 # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'. 

383 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

384 if len(diffIndex) != 0: 

385 limits = [minY] # put the minimum first 

386 for gapIndex in diffIndex: 

387 limits.append(multipleY[gapIndex]) 

388 limits.append(multipleY[gapIndex+1]) 

389 limits.append(maxY) # maximum last 

390 assert len(limits)%2 == 0, 'limits is even by design, but check anyways' 

391 for i in np.arange(0, len(limits)-1, 2): 

392 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

393 defects.append(s) 

394 else: # No gap is large enough 

395 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

396 defects.append(s) 

397 return defects 

398 

399 def debugView(self, stepname, ampImage, defects, detector): 

400 # def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover 

401 """Plot the defects found by the task. 

402 

403 Parameters 

404 ---------- 

405 exp : `lsst.afw.image.exposure.Exposure` 

406 The exposure in which the defects were found. 

407 visit : `int` 

408 The visit number. 

409 defects : `lsst.ip.isr.Defect` 

410 The defects to plot. 

411 imageType : `str` 

412 The type of image, either 'dark' or 'flat'. 

413 """ 

414 frame = getDebugFrame(self._display, stepname) 

415 if frame: 

416 disp = afwDisplay.Display(frame=frame) 

417 disp.scale('asinh', 'zscale') 

418 disp.setMaskTransparency(80) 

419 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

420 

421 maskedIm = ampImage.clone() 

422 defects.maskPixels(maskedIm, "BAD") 

423 

424 mpDict = maskedIm.mask.getMaskPlaneDict() 

425 for plane in mpDict.keys(): 

426 if plane in ['BAD']: 

427 continue 

428 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

429 

430 disp.setImageColormap('gray') 

431 disp.mtv(maskedIm) 

432 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

433 prompt = "Press Enter to continue [c]... " 

434 while True: 

435 ans = input(prompt).lower() 

436 if ans in ('', 'c', ): 

437 break 

438 

439 def debugHistogram(self, stepname, ampImage, nSigmaUsed, exp): 

440 """ 

441 Make a histogram of the distribution of pixel values for each amp. 

442 

443 The main image data histogram is plotted in blue. Edge pixels, 

444 if masked, are in red. Note that masked edge pixels do not contribute 

445 to the underflow and overflow numbers. 

446 

447 Note that this currently only supports the 16-amp LSST detectors. 

448 

449 Parameters 

450 ---------- 

451 dataRef : `lsst.daf.persistence.ButlerDataRef` 

452 dataRef for the detector. 

453 exp : `lsst.afw.image.exposure.Exposure` 

454 The exposure in which the defects were found. 

455 visit : `int` 

456 The visit number. 

457 nSigmaUsed : `float` 

458 The number of sigma used for detection 

459 """ 

460 frame = getDebugFrame(self._display, stepname) 

461 if frame: 

462 import matplotlib.pyplot as plt 

463 

464 detector = exp.getDetector() 

465 nX = np.floor(np.sqrt(len(detector))) 

466 nY = len(detector) // nX 

467 fig, ax = plt.subplots(nrows=nY, ncols=nX, sharex='col', sharey='row', figsize=(13, 10)) 

468 

469 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

470 

471 for (amp, a) in zip(reversed(detector), ax.flatten()): 

472 mi = exp.maskedImage[amp.getBBox()] 

473 

474 # normalize by expTime as we plot in ADU/s and don't always work with master calibs 

475 mi.image.array /= expTime 

476 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

477 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

478 # Get array of pixels 

479 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

480 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

481 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

482 

483 thrUpper = mean + nSigmaUsed*sigma 

484 thrLower = mean - nSigmaUsed*sigma 

485 

486 nRight = len(imgData[imgData > thrUpper]) 

487 nLeft = len(imgData[imgData < thrLower]) 

488 

489 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

490 leftEdge = mean - nsig * nSigmaUsed*sigma 

491 rightEdge = mean + nsig * nSigmaUsed*sigma 

492 nbins = np.linspace(leftEdge, rightEdge, 1000) 

493 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, 

494 lw=1, edgecolor='red') 

495 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, 

496 lw=3, edgecolor='blue') 

497 

498 # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram 

499 nOverflow = len(imgData[imgData > rightEdge]) 

500 nUnderflow = len(imgData[imgData < leftEdge]) 

501 

502 # Put v-lines and textboxes in 

503 a.axvline(thrUpper, c='k') 

504 a.axvline(thrLower, c='k') 

505 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

506 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

507 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

508 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

509 

510 # set axis limits and scales 

511 a.set_ylim([1., 1.7*np.max(y)]) 

512 lPlot, rPlot = a.get_xlim() 

513 a.set_xlim(np.array([lPlot, rPlot])) 

514 a.set_yscale('log') 

515 a.set_xlabel("ADU/s") 

516 return 

517 

518 

519class MergeDefectsConnections(pipeBase.PipelineTaskConnections, 

520 dimensions=("instrument", "detector")): 

521 inputDefects = cT.Input( 

522 name="singleExpDefects", 

523 doc="Measured defect lists.", 

524 storageClass="Defects", 

525 dimensions=("instrument", "detector", "exposure"), 

526 multiple=True, 

527 ) 

528 camera = cT.PrerequisiteInput( 

529 name='camera', 

530 doc="Camera associated with these defects.", 

531 storageClass="Camera", 

532 dimensions=("instrument", ), 

533 isCalibration=True, 

534 lookupFunction=lookupStaticCalibration, 

535 ) 

536 

537 mergedDefects = cT.Output( 

538 name="defects", 

539 doc="Final merged defects.", 

540 storageClass="Defects", 

541 dimensions=("instrument", "detector"), 

542 multiple=False, 

543 isCalibration=True, 

544 ) 

545 

546 

547class MergeDefectsTaskConfig(pipeBase.PipelineTaskConfig, 

548 pipelineConnections=MergeDefectsConnections): 

549 """Configuration for merging single exposure defects. 

550 """ 

551 assertSameRun = pexConfig.Field( 

552 dtype=bool, 

553 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or" 

554 "if the run key isn't found."), 

555 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

556 ) 

557 ignoreFilters = pexConfig.Field( 

558 dtype=bool, 

559 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

560 " images. Allows mixing of filters in the input flats. Set to False if you think" 

561 " your defects might be chromatic and want to have registry support for varying" 

562 " defects with respect to filter."), 

563 default=True, 

564 ) 

565 nullFilterName = pexConfig.Field( 

566 dtype=str, 

567 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

568 default="NONE", 

569 ) 

570 combinationMode = pexConfig.ChoiceField( 

571 doc="Which types of defects to identify", 

572 dtype=str, 

573 default="FRACTION", 

574 allowed={ 

575 "AND": "Logical AND the pixels found in each visit to form set ", 

576 "OR": "Logical OR the pixels found in each visit to form set ", 

577 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

578 } 

579 ) 

580 combinationFraction = pexConfig.RangeField( 

581 dtype=float, 

582 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

583 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

584 " mode AND to require pixel to appear in all images."), 

585 default=0.7, 

586 min=0, 

587 max=1, 

588 ) 

589 edgesAsDefects = pexConfig.Field( 

590 dtype=bool, 

591 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

592 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

593 " defect will be located there."), 

594 default=False, 

595 ) 

596 

597 

598class MergeDefectsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

599 """Merge the defects from multiple exposures. 

600 """ 

601 ConfigClass = MergeDefectsTaskConfig 

602 _DefaultName = 'cpDefectMerge' 

603 

604 def run(self, inputDefects, camera): 

605 detectorId = inputDefects[0].getMetadata().get('DETECTOR', None) 

606 if detectorId is None: 

607 raise RuntimeError("Cannot identify detector id.") 

608 detector = camera[detectorId] 

609 

610 imageTypes = set() 

611 for inDefect in inputDefects: 

612 imageType = inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN') 

613 imageTypes.add(imageType) 

614 

615 # Determine common defect pixels separately for each input image type. 

616 splitDefects = list() 

617 for imageType in imageTypes: 

618 sumImage = afwImage.MaskedImageF(detector.getBBox()) 

619 count = 0 

620 for inDefect in inputDefects: 

621 if imageType == inDefect.getMetadata().get('cpDefectGenImageType', 'UNKNOWN'): 

622 count += 1 

623 for defect in inDefect: 

624 sumImage.image[defect.getBBox()] += 1.0 

625 sumImage /= count 

626 nDetected = len(np.where(sumImage.getImage().getArray() > 0)[0]) 

627 self.log.info("Pre-merge %s pixels with non-zero detections for %s" % (nDetected, imageType)) 

628 

629 if self.config.combinationMode == 'AND': 

630 threshold = 1.0 

631 elif self.config.combinationMode == 'OR': 

632 threshold = 0.0 

633 elif self.config.combinationMode == 'FRACTION': 

634 threshold = self.config.combinationFraction 

635 else: 

636 raise RuntimeError(f"Got unsupported combinationMode {self.config.combinationMode}") 

637 indices = np.where(sumImage.getImage().getArray() > threshold) 

638 BADBIT = sumImage.getMask().getPlaneBitMask('BAD') 

639 sumImage.getMask().getArray()[indices] |= BADBIT 

640 self.log.info("Post-merge %s pixels marked as defects for %s" % (len(indices[0]), imageType)) 

641 partialDefect = Defects.fromMask(sumImage, 'BAD') 

642 splitDefects.append(partialDefect) 

643 

644 # Do final combination of separate image types 

645 finalImage = afwImage.MaskedImageF(detector.getBBox()) 

646 for inDefect in splitDefects: 

647 for defect in inDefect: 

648 finalImage.image[defect.getBBox()] += 1 

649 finalImage /= len(splitDefects) 

650 nDetected = len(np.where(finalImage.getImage().getArray() > 0)[0]) 

651 self.log.info("Pre-final merge %s pixels with non-zero detections" % (nDetected, )) 

652 

653 # This combination is the OR of all image types 

654 threshold = 0.0 

655 indices = np.where(finalImage.getImage().getArray() > threshold) 

656 BADBIT = finalImage.getMask().getPlaneBitMask('BAD') 

657 finalImage.getMask().getArray()[indices] |= BADBIT 

658 self.log.info("Post-final merge %s pixels marked as defects" % (len(indices[0]), )) 

659 

660 if self.config.edgesAsDefects: 

661 self.log.info("Masking edge pixels as defects.") 

662 # Do the same as IsrTask.maskEdges() 

663 box = detector.getBBox() 

664 subImage = finalImage[box] 

665 box.grow(-self.nPixBorder) 

666 SourceDetectionTask.setEdgeBits(subImage, box, BADBIT) 

667 

668 merged = Defects.fromMask(finalImage, 'BAD') 

669 merged.updateMetadata(camera=camera, detector=detector, filterName=None, 

670 setCalibId=True, setDate=True) 

671 

672 return pipeBase.Struct( 

673 mergedDefects=merged, 

674 ) 

675 

676 

677class FindDefectsTaskConfig(pexConfig.Config): 

678 measure = pexConfig.ConfigurableField( 

679 target=MeasureDefectsTask, 

680 doc="Task to measure single frame defects.", 

681 ) 

682 merge = pexConfig.ConfigurableField( 

683 target=MergeDefectsTask, 

684 doc="Task to merge multiple defects together.", 

685 ) 

686 

687 isrForFlats = pexConfig.ConfigurableField( 

688 target=IsrTask, 

689 doc="Task to perform instrumental signature removal", 

690 ) 

691 isrForDarks = pexConfig.ConfigurableField( 

692 target=IsrTask, 

693 doc="Task to perform instrumental signature removal", 

694 ) 

695 isrMandatoryStepsFlats = pexConfig.ListField( 

696 dtype=str, 

697 doc=("isr operations that must be performed for valid results when using flats." 

698 " Raises if any of these are False"), 

699 default=['doAssembleCcd', 'doFringe'] 

700 ) 

701 isrMandatoryStepsDarks = pexConfig.ListField( 

702 dtype=str, 

703 doc=("isr operations that must be performed for valid results when using darks. " 

704 "Raises if any of these are False"), 

705 default=['doAssembleCcd', 'doFringe'] 

706 ) 

707 isrForbiddenStepsFlats = pexConfig.ListField( 

708 dtype=str, 

709 doc=("isr operations that must NOT be performed for valid results when using flats." 

710 " Raises if any of these are True"), 

711 default=['doBrighterFatter', 'doUseOpticsTransmission', 

712 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

713 ) 

714 isrForbiddenStepsDarks = pexConfig.ListField( 

715 dtype=str, 

716 doc=("isr operations that must NOT be performed for valid results when using darks." 

717 " Raises if any of these are True"), 

718 default=['doBrighterFatter', 'doUseOpticsTransmission', 

719 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

720 ) 

721 isrDesirableSteps = pexConfig.ListField( 

722 dtype=str, 

723 doc=("isr operations that it is advisable to perform, but are not mission-critical." 

724 " WARNs are logged for any of these found to be False."), 

725 default=['doBias'] 

726 ) 

727 

728 ccdKey = pexConfig.Field( 

729 dtype=str, 

730 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'", 

731 default='ccd', 

732 ) 

733 imageTypeKey = pexConfig.Field( 

734 dtype=str, 

735 doc="The key for the butler to use by which to check whether images are darks or flats", 

736 default='imageType', 

737 ) 

738 

739 

740class FindDefectsTask(pipeBase.CmdLineTask): 

741 """Task for finding defects in sensors. 

742 

743 The task has two modes of operation, defect finding in raws and in 

744 master calibrations, which work as follows. 

745 

746 Master calib defect finding 

747 ---------------------------- 

748 

749 A single visit number is supplied, for which the corresponding flat & dark 

750 will be used. This is because, at present at least, there is no way to pass 

751 a calibration exposure ID from the command line to a command line task. 

752 

753 The task retrieves the corresponding dark and flat exposures for the 

754 supplied visit. If a flat is available the task will (be able to) look 

755 for both bright and dark defects. If only a dark is found then only bright 

756 defects will be sought. 

757 

758 All pixels above/below the specified nSigma which lie with the specified 

759 borders for flats/darks are identified as defects. 

760 

761 Raw visit defect finding 

762 ------------------------ 

763 

764 A list of exposure IDs are supplied for defect finding. The task will 

765 detect bright pixels in the dark frames, if supplied, and bright & dark 

766 pixels in the flats, if supplied, i.e. if you only supply darks you will 

767 only be given bright defects. This is done automatically from the imageType 

768 of the exposure, so the input exposure list can be a mix. 

769 

770 As with the master calib detection, all pixels above/below the specified 

771 nSigma which lie with the specified borders for flats/darks are identified 

772 as defects. Then, a post-processing step is done to merge these detections, 

773 with pixels appearing in a fraction [0..1] of the images are kept as defects 

774 and those appearing below that occurrence-threshold are discarded. 

775 """ 

776 ConfigClass = FindDefectsTaskConfig 

777 _DefaultName = "findDefects" 

778 

779 RunnerClass = DataRefListRunner 

780 

781 def __init__(self, **kwargs): 

782 super().__init__(**kwargs) 

783 self.makeSubtask("measure") 

784 self.makeSubtask("merge") 

785 

786 @pipeBase.timeMethod 

787 def runDataRef(self, dataRefList): 

788 """Run the defect finding task. 

789 

790 Find the defects, as described in the main task docstring, from a 

791 dataRef and a list of visit(s). 

792 

793 Parameters 

794 ---------- 

795 dataRefList : `list` [`lsst.daf.persistence.ButlerDataRef`] 

796 dataRefs for the data to be checked for defects. 

797 

798 Returns 

799 ------- 

800 result : `lsst.pipe.base.Struct` 

801 Result struct with Components: 

802 

803 - ``defects`` : `lsst.ip.isr.Defect` 

804 The defects found by the task. 

805 - ``exitStatus`` : `int` 

806 The exit code. 

807 """ 

808 dataRef = dataRefList[0] 

809 camera = dataRef.get("camera") 

810 

811 singleExpDefects = [] 

812 activeChip = None 

813 for dataRef in dataRefList: 

814 exposure = dataRef.get("postISRCCD") 

815 if activeChip: 

816 if exposure.getDetector().getName() != activeChip: 

817 raise RuntimeError("Too many input detectors supplied!") 

818 else: 

819 activeChip = exposure.getDetector().getName() 

820 

821 result = self.measure.run(exposure, camera) 

822 singleExpDefects.append(result.outputDefects) 

823 

824 finalResults = self.merge.run(singleExpDefects, camera) 

825 metadata = finalResults.mergedDefects.getMetadata() 

826 inputDims = {'calibDate': metadata['CALIBDATE'], 

827 'raftName': metadata['RAFTNAME'], 

828 'detectorName': metadata['SLOTNAME'], 

829 'detector': metadata['DETECTOR'], 

830 'ccd': metadata['DETECTOR'], 

831 'ccdnum': metadata['DETECTOR']} 

832 

833 butler = dataRef.getButler() 

834 butler.put(finalResults.mergedDefects, "defects", inputDims) 

835 

836 return finalResults