Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['FindDefectsTask', 

24 'FindDefectsTaskConfig', ] 

25 

26import numpy as np 

27import os 

28import warnings 

29 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.afw.image as afwImage 

33import lsst.meas.algorithms as measAlg 

34import lsst.afw.math as afwMath 

35import lsst.afw.detection as afwDetection 

36import lsst.afw.display as afwDisplay 

37from lsst.afw import cameraGeom 

38from lsst.geom import Box2I, Point2I 

39import lsst.daf.base as dafBase 

40 

41from lsst.ip.isr import IsrTask 

42from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \ 

43 validateIsrConfig 

44 

45 

46class FindDefectsTaskConfig(pexConfig.Config): 

47 """Config class for defect finding""" 

48 

49 isrForFlats = pexConfig.ConfigurableField( 

50 target=IsrTask, 

51 doc="Task to perform instrumental signature removal", 

52 ) 

53 isrForDarks = pexConfig.ConfigurableField( 

54 target=IsrTask, 

55 doc="Task to perform instrumental signature removal", 

56 ) 

57 isrMandatoryStepsFlats = pexConfig.ListField( 

58 dtype=str, 

59 doc=("isr operations that must be performed for valid results when using flats." 

60 " Raises if any of these are False"), 

61 default=['doAssembleCcd', 'doFringe'] 

62 ) 

63 isrMandatoryStepsDarks = pexConfig.ListField( 

64 dtype=str, 

65 doc=("isr operations that must be performed for valid results when using darks. " 

66 "Raises if any of these are False"), 

67 default=['doAssembleCcd', 'doFringe'] 

68 ) 

69 isrForbiddenStepsFlats = pexConfig.ListField( 

70 dtype=str, 

71 doc=("isr operations that must NOT be performed for valid results when using flats." 

72 " Raises if any of these are True"), 

73 default=['doBrighterFatter', 'doUseOpticsTransmission', 

74 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

75 ) 

76 isrForbiddenStepsDarks = pexConfig.ListField( 

77 dtype=str, 

78 doc=("isr operations that must NOT be performed for valid results when using darks." 

79 " Raises if any of these are True"), 

80 default=['doBrighterFatter', 'doUseOpticsTransmission', 

81 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

82 ) 

83 isrDesirableSteps = pexConfig.ListField( 

84 dtype=str, 

85 doc=("isr operations that it is advisable to perform, but are not mission-critical." 

86 " WARNs are logged for any of these found to be False."), 

87 default=['doBias'] 

88 ) 

89 ccdKey = pexConfig.Field( 

90 dtype=str, 

91 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'", 

92 default='ccd', 

93 ) 

94 imageTypeKey = pexConfig.Field( 

95 dtype=str, 

96 doc="The key for the butler to use by which to check whether images are darks or flats", 

97 default='imageType', 

98 ) 

99 mode = pexConfig.ChoiceField( 

100 doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?" 

101 " If MASTER, a single visit number should be supplied, for which the corresponding master flat" 

102 " and dark will be used. If VISITS, the list of visits will be used, treating the flats and " 

103 " darks as appropriate, depending on their image types, as determined by their imageType from" 

104 " config.imageTypeKey"), 

105 dtype=str, 

106 default="VISITS", 

107 allowed={ 

108 "VISITS": "Calculate defects from a list of raw visits", 

109 "MASTER": "Use the corresponding master calibs from the specified visit to measure defects", 

110 } 

111 ) 

112 nSigmaBright = pexConfig.Field( 

113 dtype=float, 

114 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be", 

115 " appropriate for some LSST sensors in DM-17490."), 

116 default=4.8, 

117 ) 

118 nSigmaDark = pexConfig.Field( 

119 dtype=float, 

120 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be", 

121 " appropriate for some LSST sensors in DM-17490."), 

122 default=5.0, 

123 ) 

124 nPixBorderUpDown = pexConfig.Field( 

125 dtype=int, 

126 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

127 default=7, 

128 ) 

129 nPixBorderLeftRight = pexConfig.Field( 

130 dtype=int, 

131 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

132 default=7, 

133 ) 

134 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

135 dtype=int, 

136 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ", 

137 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ", 

138 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ", 

139 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ", 

140 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ", 

141 "non-bad pixels in BPC, an exception to the above is made and those consecutive ", 

142 "'goodPixelColumnGapThreshold' are not marked as bad."), 

143 default=50, 

144 ) 

145 goodPixelColumnGapThreshold = pexConfig.Field( 

146 dtype=int, 

147 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ", 

148 "'badOnAndOffPixelColumnThreshold')."), 

149 default=30, 

150 ) 

151 edgesAsDefects = pexConfig.Field( 

152 dtype=bool, 

153 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

154 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

155 " defect will be located there."), 

156 default=False, 

157 ) 

158 assertSameRun = pexConfig.Field( 

159 dtype=bool, 

160 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or" 

161 "if the run key isn't found."), 

162 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

163 ) 

164 ignoreFilters = pexConfig.Field( 

165 dtype=bool, 

166 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

167 " images. Allows mixing of filters in the input flats. Set to False if you think" 

168 " your defects might be chromatic and want to have registry support for varying" 

169 " defects with respect to filter."), 

170 default=True, 

171 ) 

172 nullFilterName = pexConfig.Field( 

173 dtype=str, 

174 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

175 default="NONE", 

176 ) 

177 combinationMode = pexConfig.ChoiceField( 

178 doc="Which types of defects to identify", 

179 dtype=str, 

180 default="FRACTION", 

181 allowed={ 

182 "AND": "Logical AND the pixels found in each visit to form set ", 

183 "OR": "Logical OR the pixels found in each visit to form set ", 

184 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

185 } 

186 ) 

187 combinationFraction = pexConfig.RangeField( 

188 dtype=float, 

189 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

190 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

191 " mode AND to require pixel to appear in all images."), 

192 default=0.7, 

193 min=0, 

194 max=1, 

195 ) 

196 makePlots = pexConfig.Field( 

197 dtype=bool, 

198 doc=("Plot histograms for each visit for each amp (one plot per detector) and the final" 

199 " defects overlaid on the sensor."), 

200 default=False, 

201 ) 

202 writeAs = pexConfig.ChoiceField( 

203 doc="Write the output file as ASCII or FITS table", 

204 dtype=str, 

205 default="FITS", 

206 allowed={ 

207 "ASCII": "Write the output as an ASCII file", 

208 "FITS": "Write the output as an FITS table", 

209 "BOTH": "Write the output as both a FITS table and an ASCII file", 

210 } 

211 ) 

212 

213 

214class FindDefectsTask(pipeBase.CmdLineTask): 

215 """Task for finding defects in sensors. 

216 

217 The task has two modes of operation, defect finding in raws and in 

218 master calibrations, which work as follows. 

219 

220 Master calib defect finding 

221 ---------------------------- 

222 

223 A single visit number is supplied, for which the corresponding flat & dark 

224 will be used. This is because, at present at least, there is no way to pass 

225 a calibration exposure ID from the command line to a command line task. 

226 

227 The task retrieves the corresponding dark and flat exposures for the 

228 supplied visit. If a flat is available the task will (be able to) look 

229 for both bright and dark defects. If only a dark is found then only bright 

230 defects will be sought. 

231 

232 All pixels above/below the specified nSigma which lie with the specified 

233 borders for flats/darks are identified as defects. 

234 

235 Raw visit defect finding 

236 ------------------------ 

237 

238 A list of exposure IDs are supplied for defect finding. The task will 

239 detect bright pixels in the dark frames, if supplied, and bright & dark 

240 pixels in the flats, if supplied, i.e. if you only supply darks you will 

241 only be given bright defects. This is done automatically from the imageType 

242 of the exposure, so the input exposure list can be a mix. 

243 

244 As with the master calib detection, all pixels above/below the specified 

245 nSigma which lie with the specified borders for flats/darks are identified 

246 as defects. Then, a post-processing step is done to merge these detections, 

247 with pixels appearing in a fraction [0..1] of the images are kept as defects 

248 and those appearing below that occurrence-threshold are discarded. 

249 """ 

250 

251 RunnerClass = SingleVisitListTaskRunner 

252 ConfigClass = FindDefectsTaskConfig 

253 _DefaultName = "findDefects" 

254 

255 def __init__(self, *args, **kwargs): 

256 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

257 self.makeSubtask("isrForFlats") 

258 self.makeSubtask("isrForDarks") 

259 

260 validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats, 

261 self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps) 

262 validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks, 

263 self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps) 

264 self.config.validate() 

265 self.config.freeze() 

266 

267 @classmethod 

268 def _makeArgumentParser(cls): 

269 """Augment argument parser for the FindDefectsTask.""" 

270 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

271 parser.add_argument("--visitList", dest="visitList", nargs="*", 

272 help=("List of visits to use. Same for each detector." 

273 " Uses the normal 0..10:3^234 syntax")) 

274 parser.add_id_argument("--id", datasetType="newDefects", 

275 ContainerClass=NonexistentDatasetTaskDataIdContainer, 

276 help="The ccds to use, e.g. --id ccd=0..100") 

277 return parser 

278 

279 @pipeBase.timeMethod 

280 def runDataRef(self, dataRef, visitList): 

281 """Run the defect finding task. 

282 

283 Find the defects, as described in the main task docstring, from a 

284 dataRef and a list of visit(s). 

285 

286 Parameters 

287 ---------- 

288 dataRef : `lsst.daf.persistence.ButlerDataRef` 

289 dataRef for the detector for the visits to be fit. 

290 visitList : `list` [`int`] 

291 List of visits to be processed. If config.mode == 'VISITS' then the 

292 list of visits is used. If config.mode == 'MASTER' then the length 

293 of visitList must be one, and the corresponding master calibrations 

294 are used. 

295 

296 Returns 

297 ------- 

298 result : `lsst.pipe.base.Struct` 

299 Result struct with Components: 

300 

301 - ``defects`` : `lsst.meas.algorithms.Defect` 

302 The defects found by the task. 

303 - ``exitStatus`` : `int` 

304 The exit code. 

305 """ 

306 

307 detNum = dataRef.dataId[self.config.ccdKey] 

308 self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum)) 

309 

310 defectLists = {'dark': [], 'flat': []} 

311 

312 midTime = 0 

313 filters = set() 

314 

315 if self.config.mode == 'MASTER': 

316 if len(visitList) > 1: 

317 raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}") 

318 dataRef.dataId['expId'] = visitList[0] 

319 

320 for datasetType in defectLists.keys(): 

321 exp = dataRef.get(datasetType) 

322 midTime += self._getMjd(exp) 

323 filters.add(exp.getFilter().getName()) 

324 defects = self.findHotAndColdPixels(exp, datasetType) 

325 

326 msg = "Found %s defects containing %s pixels in master %s" 

327 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

328 defectLists[datasetType].append(defects) 

329 if self.config.makePlots: 

330 self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType), 

331 defects, datasetType) 

332 midTime /= len(defectLists.keys()) 

333 

334 elif self.config.mode == 'VISITS': 

335 butler = dataRef.getButler() 

336 

337 if self.config.assertSameRun: 

338 runs = self._getRunListFromVisits(butler, visitList) 

339 if len(runs) != 1: 

340 raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True") 

341 

342 for visit in visitList: 

343 imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'expId': visit})[0] 

344 imageType = imageType.lower() 

345 dataRef.dataId['expId'] = visit 

346 

347 if imageType == 'flat': # note different isr tasks 

348 exp = self.isrForFlats.runDataRef(dataRef).exposure 

349 defects = self.findHotAndColdPixels(exp, imageType) 

350 defectLists['flat'].append(defects) 

351 midTime += self._getMjd(exp) 

352 filters.add(exp.getFilter().getName()) 

353 

354 elif imageType == 'dark': 

355 exp = self.isrForDarks.runDataRef(dataRef).exposure 

356 defects = self.findHotAndColdPixels(exp, imageType) 

357 defectLists['dark'].append(defects) 

358 midTime += self._getMjd(exp) 

359 filters.add(exp.getFilter().getName()) 

360 

361 else: 

362 raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported") 

363 

364 msg = "Found %s defects containing %s pixels in visit %s" 

365 self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit) 

366 

367 if self.config.makePlots: 

368 self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType) 

369 

370 midTime /= len(visitList) 

371 

372 msg = "Combining %s defect sets from darks for detector %s" 

373 self.log.info(msg, len(defectLists['dark']), detNum) 

374 mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(), 

375 self.config.combinationMode) 

376 msg = "Combining %s defect sets from flats for detector %s" 

377 self.log.info(msg, len(defectLists['flat']), detNum) 

378 mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(), 

379 self.config.combinationMode) 

380 

381 msg = "Combining bright and dark defect sets for detector %s" 

382 self.log.info(msg, detNum) 

383 brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats] 

384 allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR') 

385 

386 self._writeData(dataRef, allDefects, midTime, filters) 

387 

388 self.log.info("Finished finding defects in detector %s" % detNum) 

389 return pipeBase.Struct(defects=allDefects, exitStatus=0) 

390 

391 def _getNsigmaForPlot(self, imageType): 

392 assert imageType in ['flat', 'dark'] 

393 nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark 

394 return nSig 

395 

396 @staticmethod 

397 def _nPixFromDefects(defect): 

398 """Count the number of pixels in a defect object.""" 

399 nPix = 0 

400 for d in defect: 

401 nPix += d.getBBox().getArea() 

402 return nPix 

403 

404 def _writeData(self, dataRef, defects, midTime, filters): 

405 """Write the data out to the defect file. 

406 

407 Parameters 

408 ---------- 

409 dataRef : `lsst.daf.persistence.ButlerDataRef` 

410 dataRef for the detector for defects to be written. 

411 defects : `lsst.meas.algorithms.Defect` 

412 The defects to be written. 

413 """ 

414 date = dafBase.DateTime(midTime, dafBase.DateTime.MJD).toPython().isoformat() 

415 

416 detName = self._getDetectorName(dataRef) 

417 instrumentName = self._getInstrumentName(dataRef) 

418 detNum = self._getDetectorNumber(dataRef) 

419 if not self.config.ignoreFilters: 

420 filt = self._filterSetToFilterString(filters) 

421 else: 

422 filt = self.config.nullFilterName 

423 

424 CALIB_ID = f"detectorName={detName} detector={detNum} calibDate={date} ccd={detNum} filter={filt}" 

425 try: 

426 raftName = detName.split("_")[0] 

427 CALIB_ID += f" raftName={raftName}" 

428 except Exception: 

429 pass 

430 

431 now = dafBase.DateTime.now().toPython() 

432 mdOriginal = defects.getMetadata() 

433 mdSupplemental = {"INSTRUME": instrumentName, 

434 "DETECTOR": dataRef.dataId['detector'], 

435 "CALIBDATE": date, 

436 "CALIB_ID": CALIB_ID, 

437 "CALIB_CREATION_DATE": now.date().isoformat(), 

438 "CALIB_CREATION_TIME": now.time().isoformat()} 

439 

440 mdOriginal.update(mdSupplemental) 

441 

442 # TODO: DM-23508 sort out the butler abuse from here-on down in Gen3 

443 # defects should simply be butler.put() 

444 templateFilename = dataRef.getUri(write=True) # does not guarantee that full path exists 

445 baseDirName = os.path.dirname(templateFilename) 

446 # ingest curated calibs demands detectorName is lowercase 

447 dirName = os.path.join(baseDirName, instrumentName, "defects", detName.lower()) 

448 if not os.path.exists(dirName): 

449 os.makedirs(dirName) 

450 

451 date += ".fits" 

452 filename = os.path.join(dirName, date) 

453 

454 msg = "Writing defects to %s in format: %s" 

455 self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs) 

456 if self.config.writeAs in ['FITS', 'BOTH']: 

457 defects.writeFits(filename) 

458 if self.config.writeAs in ['ASCII', 'BOTH']: 

459 wroteTo = defects.writeText(filename) 

460 assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0]) 

461 return 

462 

463 @staticmethod 

464 def _filterSetToFilterString(filters): 

465 return "~".join([f for f in filters]) 

466 

467 @staticmethod 

468 def _getDetectorNumber(dataRef): 

469 dataRefDetNum = dataRef.dataId['detector'] 

470 camera = dataRef.get('camera') 

471 detectorDetNum = camera[dataRef.dataId['detector']].getId() 

472 assert dataRefDetNum == detectorDetNum 

473 return dataRefDetNum 

474 

475 @staticmethod 

476 def _getInstrumentName(dataRef): 

477 camera = dataRef.get('camera') 

478 return camera.getName() 

479 

480 @staticmethod 

481 def _getDetectorName(dataRef): 

482 camera = dataRef.get('camera') 

483 return camera[dataRef.dataId['detector']].getName() 

484 

485 @staticmethod 

486 def _getMjd(exp, timescale=dafBase.DateTime.UTC): 

487 vi = exp.getInfo().getVisitInfo() 

488 dateObs = vi.getDate() 

489 mjd = dateObs.get(dafBase.DateTime.MJD) 

490 return mjd 

491 

492 @staticmethod 

493 def _getRunListFromVisits(butler, visitList): 

494 """Return the set of runs for the visits in visitList.""" 

495 runs = set() 

496 for visit in visitList: 

497 runs.add(butler.queryMetadata('raw', 'run', dataId={'expId': visit})[0]) 

498 return runs 

499 

500 def _postProcessDefectSets(self, defectList, imageDimensions, mode): 

501 """Combine a list of defects to make a single defect object. 

502 

503 AND, OR or use percentage of visits in which defects appear 

504 depending on config. 

505 

506 Parameters 

507 ---------- 

508 defectList : `list` [`lsst.meas.algorithms.Defect`] 

509 The lList of defects to merge. 

510 imageDimensions : `tuple` [`int`] 

511 The size of the image. 

512 mode : `str` 

513 The combination mode to use, either 'AND', 'OR' or 'FRACTION' 

514 

515 Returns 

516 ------- 

517 defects : `lsst.meas.algorithms.Defect` 

518 The defect set resulting from the merge. 

519 """ 

520 # so that empty lists can be passed in for input data 

521 # where only flats or darks are supplied 

522 if defectList == []: 

523 return [] 

524 

525 if len(defectList) == 1: # single input - no merging to do 

526 return defectList[0] 

527 

528 sumImage = afwImage.MaskedImageF(imageDimensions) 

529 for defects in defectList: 

530 for defect in defects: 

531 sumImage.image[defect.getBBox()] += 1 

532 sumImage /= len(defectList) 

533 

534 nDetected = len(np.where(sumImage.image.array > 0)[0]) 

535 self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected) 

536 

537 if mode == 'OR': # must appear in any 

538 indices = np.where(sumImage.image.array > 0) 

539 else: 

540 if mode == 'AND': # must appear in all 

541 threshold = 1 

542 elif mode == 'FRACTION': 

543 threshold = self.config.combinationFraction 

544 else: 

545 raise RuntimeError(f"Got unsupported combinationMode {mode}") 

546 indices = np.where(sumImage.image.array >= threshold) 

547 

548 BADBIT = sumImage.mask.getPlaneBitMask('BAD') 

549 sumImage.mask.array[indices] |= BADBIT 

550 

551 self.log.info("Post-merge %s pixels marked as defects" % len(indices[0])) 

552 

553 if self.config.edgesAsDefects: 

554 self.log.info("Masking edge pixels as defects in addition to previously identified defects") 

555 self._setEdgeBits(sumImage, 'BAD') 

556 

557 defects = measAlg.Defects.fromMask(sumImage, 'BAD') 

558 return defects 

559 

560 @staticmethod 

561 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

562 """Return the number of non-bad pixels in the image.""" 

563 nPixels = maskedIm.mask.array.size 

564 nBad = countMaskedPixels(maskedIm, badMaskString) 

565 return nPixels - nBad 

566 

567 def findHotAndColdPixels(self, exp, imageType, setMask=False): 

568 """Find hot and cold pixels in an image. 

569 

570 Using config-defined thresholds on a per-amp basis, mask pixels 

571 that are nSigma above threshold in dark frames (hot pixels), 

572 or nSigma away from the clipped mean in flats (hot & cold pixels). 

573 

574 Parameters 

575 ---------- 

576 exp : `lsst.afw.image.exposure.Exposure` 

577 The exposure in which to find defects. 

578 imageType : `str` 

579 The image type, either 'dark' or 'flat'. 

580 setMask : `bool` 

581 If true, update exp with hot and cold pixels. 

582 hot: DETECTED 

583 cold: DETECTED_NEGATIVE 

584 

585 Returns 

586 ------- 

587 defects : `lsst.meas.algorithms.Defect` 

588 The defects found in the image. 

589 """ 

590 assert imageType in ['flat', 'dark'] 

591 

592 self._setEdgeBits(exp) 

593 maskedIm = exp.maskedImage 

594 

595 # the detection polarity for afwDetection, True for positive, 

596 # False for negative, and therefore True for darks as they only have 

597 # bright pixels, and both for flats, as they have bright and dark pix 

598 polarities = {'dark': [True], 'flat': [True, False]}[imageType] 

599 

600 footprintList = [] 

601 

602 for amp in exp.getDetector(): 

603 ampImg = maskedIm[amp.getBBox()].clone() 

604 

605 # crop ampImage depending on where the amp lies in the image 

606 if self.config.nPixBorderLeftRight: 

607 if ampImg.getX0() == 0: 

608 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

609 else: 

610 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

611 if self.config.nPixBorderUpDown: 

612 if ampImg.getY0() == 0: 

613 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

614 else: 

615 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

616 

617 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

618 continue 

619 

620 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

621 

622 mergedSet = None 

623 for polarity in polarities: 

624 nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark 

625 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

626 

627 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

628 if setMask: 

629 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

630 

631 if mergedSet is None: 

632 mergedSet = footprintSet 

633 else: 

634 mergedSet.merge(footprintSet) 

635 

636 footprintList += mergedSet.getFootprints() 

637 

638 defects = measAlg.Defects.fromFootprintList(footprintList) 

639 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

640 

641 return defects 

642 

643 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

644 """Mask blocks in a column if there are on-and-off bad pixels 

645 

646 If there's a column with on and off bad pixels, mask all the pixels in between, 

647 except if there is a large enough gap of consecutive good pixels between two 

648 bad pixels in the column. 

649 

650 Parameters 

651 --------- 

652 defects: `lsst.meas.algorithms.Defect` 

653 The defects found in the image so far 

654 

655 Returns 

656 ------ 

657 defects: `lsst.meas.algorithms.Defect` 

658 If the number of bad pixels in a column is not larger or equal than 

659 self.config.badPixelColumnThreshold, the iput list is returned. Otherwise, 

660 the defects list returned will include boxes that mask blocks of on-and-of 

661 pixels. 

662 """ 

663 # Get the (x, y) values of each bad pixel in amp. 

664 coordinates = [] 

665 for defect in defects: 

666 bbox = defect.getBBox() 

667 x0, y0 = bbox.getMinX(), bbox.getMinY() 

668 deltaX0, deltaY0 = bbox.getDimensions() 

669 for j in np.arange(y0, y0+deltaY0): 

670 for i in np.arange(x0, x0 + deltaX0): 

671 coordinates.append((i, j)) 

672 

673 x, y = [], [] 

674 for coordinatePair in coordinates: 

675 x.append(coordinatePair[0]) 

676 y.append(coordinatePair[1]) 

677 

678 x = np.array(x) 

679 y = np.array(y) 

680 # Find the defects with same "x" (vertical) coordinate (column). 

681 unique, counts = np.unique(x, return_counts=True) 

682 multipleX = [] 

683 for (a, b) in zip(unique, counts): 

684 if b >= self.config.badOnAndOffPixelColumnThreshold: 

685 multipleX.append(a) 

686 if len(multipleX) != 0: 

687 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

688 

689 return defects 

690 

691 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

692 """Mask blocks in a column if number of on-and-off bad pixels is above threshold. 

693 

694 This function is called if the number of on-and-off bad pixels in a column 

695 is larger or equal than self.config.badOnAndOffPixelColumnThreshold. 

696 

697 Parameters 

698 --------- 

699 x: list 

700 Lower left x coordinate of defect box. x coordinate is along the short axis if amp. 

701 

702 y: list 

703 Lower left y coordinate of defect box. x coordinate is along the long axis if amp. 

704 

705 multipleX: list 

706 List of x coordinates in amp. with multiple bad pixels (i.e., columns with defects). 

707 

708 defects: `lsst.meas.algorithms.Defect` 

709 The defcts found in the image so far 

710 

711 Returns 

712 ------- 

713 defects: `lsst.meas.algorithms.Defect` 

714 The defects list returned that will include boxes that mask blocks 

715 of on-and-of pixels. 

716 """ 

717 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

718 for x0 in multipleX: 

719 index = np.where(x == x0) 

720 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

721 minY, maxY = np.min(multipleY), np.max(multipleY) 

722 # Next few lines: don't mask pixels in column if gap of good pixels between 

723 # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'. 

724 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

725 if len(diffIndex) != 0: 

726 limits = [minY] # put the minimum first 

727 for gapIndex in diffIndex: 

728 limits.append(multipleY[gapIndex]) 

729 limits.append(multipleY[gapIndex+1]) 

730 limits.append(maxY) # maximum last 

731 assert len(limits)%2 == 0, 'limits is even by design, but check anyways' 

732 for i in np.arange(0, len(limits)-1, 2): 

733 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

734 if s not in defects: 

735 defects.append(s) 

736 else: # No gap is large enough 

737 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

738 if s not in defects: 

739 defects.append(s) 

740 return defects 

741 

742 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

743 """Set edge bits on an exposure or maskedImage. 

744 

745 Raises 

746 ------ 

747 TypeError 

748 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

749 """ 

750 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

751 mi = exposureOrMaskedImage.maskedImage 

752 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

753 mi = exposureOrMaskedImage 

754 else: 

755 t = type(exposureOrMaskedImage) 

756 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

757 

758 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

759 if self.config.nPixBorderLeftRight: 

760 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

761 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

762 if self.config.nPixBorderUpDown: 

763 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

764 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

765 

766 def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover 

767 """Plot the defects and pixel histograms. 

768 

769 Parameters 

770 ---------- 

771 dataRef : `lsst.daf.persistence.ButlerDataRef` 

772 dataRef for the detector. 

773 exp : `lsst.afw.image.exposure.Exposure` 

774 The exposure in which the defects were found. 

775 visit : `int` 

776 The visit number. 

777 nSig : `float` 

778 The number of sigma used for detection 

779 defects : `lsst.meas.algorithms.Defect` 

780 The defects to plot. 

781 imageType : `str` 

782 The type of image, either 'dark' or 'flat'. 

783 

784 Currently only for LSST sensors. Plots are written to the path 

785 given by the butler for the ``cpPipePlotRoot`` dataset type. 

786 """ 

787 import matplotlib.pyplot as plt 

788 from matplotlib.backends.backend_pdf import PdfPages 

789 

790 afwDisplay.setDefaultBackend("matplotlib") 

791 plt.interactive(False) # seems to need reasserting here 

792 

793 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True) 

794 if not os.path.exists(dirname): 

795 os.makedirs(dirname) 

796 

797 detNum = exp.getDetector().getId() 

798 nAmps = len(exp.getDetector()) 

799 

800 if self.config.mode == "MASTER": 

801 filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf" 

802 elif self.config.mode == "VISITS": 

803 filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf" 

804 

805 filenameFull = os.path.join(dirname, filename) 

806 

807 with warnings.catch_warnings(): 

808 msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure." 

809 warnings.filterwarnings("ignore", message=msg) 

810 with PdfPages(filenameFull) as pdfPages: 

811 if nAmps == 16: 

812 self._plotAmpHistogram(dataRef, exp, visit, nSig) 

813 pdfPages.savefig() 

814 

815 self._plotDefects(exp, visit, defects, imageType) 

816 pdfPages.savefig() 

817 self.log.info("Wrote plot(s) to %s" % filenameFull) 

818 

819 def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover 

820 """Plot the defects found by the task. 

821 

822 Parameters 

823 ---------- 

824 exp : `lsst.afw.image.exposure.Exposure` 

825 The exposure in which the defects were found. 

826 visit : `int` 

827 The visit number. 

828 defects : `lsst.meas.algorithms.Defect` 

829 The defects to plot. 

830 imageType : `str` 

831 The type of image, either 'dark' or 'flat'. 

832 """ 

833 expCopy = exp.clone() # we mess with the copy later, so make a clone 

834 del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes 

835 maskedIm = expCopy.maskedImage 

836 

837 defects.maskPixels(expCopy.maskedImage, "BAD") 

838 detector = expCopy.getDetector() 

839 

840 disp = afwDisplay.Display(0, reopenPlot=True, dpi=200) 

841 

842 if imageType == "flat": # set each amp image to have a mean of 1.00 

843 for amp in detector: 

844 ampIm = maskedIm.image[amp.getBBox()] 

845 ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1 

846 

847 mpDict = maskedIm.mask.getMaskPlaneDict() 

848 for plane in mpDict.keys(): 

849 if plane in ['BAD']: 

850 continue 

851 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

852 

853 disp.scale('asinh', 'zscale') 

854 disp.setMaskTransparency(80) 

855 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

856 

857 disp.setImageColormap('gray') 

858 title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}" 

859 f", Type: {imageType}, visit: {visit}") 

860 disp.mtv(maskedIm, title=title) 

861 

862 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

863 

864 def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover 

865 """ 

866 Make a histogram of the distribution of pixel values for each amp. 

867 

868 The main image data histogram is plotted in blue. Edge pixels, 

869 if masked, are in red. Note that masked edge pixels do not contribute 

870 to the underflow and overflow numbers. 

871 

872 Note that this currently only supports the 16-amp LSST detectors. 

873 

874 Parameters 

875 ---------- 

876 dataRef : `lsst.daf.persistence.ButlerDataRef` 

877 dataRef for the detector. 

878 exp : `lsst.afw.image.exposure.Exposure` 

879 The exposure in which the defects were found. 

880 visit : `int` 

881 The visit number. 

882 nSigmaUsed : `float` 

883 The number of sigma used for detection 

884 """ 

885 import matplotlib.pyplot as plt 

886 

887 detector = exp.getDetector() 

888 

889 if len(detector) != 16: 

890 raise RuntimeError("Plotting currently only supported for 16 amp detectors") 

891 fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

892 

893 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

894 

895 for (amp, a) in zip(reversed(detector), ax.flatten()): 

896 mi = exp.maskedImage[amp.getBBox()] 

897 

898 # normalize by expTime as we plot in ADU/s and don't always work with master calibs 

899 mi.image.array /= expTime 

900 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

901 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

902 

903 # Get array of pixels 

904 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

905 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

906 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

907 

908 thrUpper = mean + nSigmaUsed*sigma 

909 thrLower = mean - nSigmaUsed*sigma 

910 

911 nRight = len(imgData[imgData > thrUpper]) 

912 nLeft = len(imgData[imgData < thrLower]) 

913 

914 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

915 leftEdge = mean - nsig * nSigmaUsed*sigma 

916 rightEdge = mean + nsig * nSigmaUsed*sigma 

917 nbins = np.linspace(leftEdge, rightEdge, 1000) 

918 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red') 

919 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue') 

920 

921 # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram 

922 nOverflow = len(imgData[imgData > rightEdge]) 

923 nUnderflow = len(imgData[imgData < leftEdge]) 

924 

925 # Put v-lines and textboxes in 

926 a.axvline(thrUpper, c='k') 

927 a.axvline(thrLower, c='k') 

928 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

929 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

930 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

931 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

932 

933 # set axis limits and scales 

934 a.set_ylim([1., 1.7*np.max(y)]) 

935 lPlot, rPlot = a.get_xlim() 

936 a.set_xlim(np.array([lPlot, rPlot])) 

937 a.set_yscale('log') 

938 a.set_xlabel("ADU/s") 

939 

940 return