Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['FindDefectsTask', 

24 'FindDefectsTaskConfig', ] 

25 

26import numpy as np 

27import os 

28import warnings 

29 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32import lsst.afw.image as afwImage 

33import lsst.meas.algorithms as measAlg 

34import lsst.afw.math as afwMath 

35import lsst.afw.detection as afwDetection 

36import lsst.afw.display as afwDisplay 

37from lsst.afw import cameraGeom 

38from lsst.geom import Box2I, Point2I 

39import lsst.daf.base as dafBase 

40 

41from lsst.ip.isr import IsrTask 

42from .utils import NonexistentDatasetTaskDataIdContainer, SingleVisitListTaskRunner, countMaskedPixels, \ 

43 validateIsrConfig 

44 

45 

46class FindDefectsTaskConfig(pexConfig.Config): 

47 """Config class for defect finding""" 

48 

49 isrForFlats = pexConfig.ConfigurableField( 

50 target=IsrTask, 

51 doc="Task to perform instrumental signature removal", 

52 ) 

53 isrForDarks = pexConfig.ConfigurableField( 

54 target=IsrTask, 

55 doc="Task to perform instrumental signature removal", 

56 ) 

57 isrMandatoryStepsFlats = pexConfig.ListField( 

58 dtype=str, 

59 doc=("isr operations that must be performed for valid results when using flats." 

60 " Raises if any of these are False"), 

61 default=['doAssembleCcd', 'doFringe'] 

62 ) 

63 isrMandatoryStepsDarks = pexConfig.ListField( 

64 dtype=str, 

65 doc=("isr operations that must be performed for valid results when using darks. " 

66 "Raises if any of these are False"), 

67 default=['doAssembleCcd', 'doFringe'] 

68 ) 

69 isrForbiddenStepsFlats = pexConfig.ListField( 

70 dtype=str, 

71 doc=("isr operations that must NOT be performed for valid results when using flats." 

72 " Raises if any of these are True"), 

73 default=['doBrighterFatter', 'doUseOpticsTransmission', 

74 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

75 ) 

76 isrForbiddenStepsDarks = pexConfig.ListField( 

77 dtype=str, 

78 doc=("isr operations that must NOT be performed for valid results when using darks." 

79 " Raises if any of these are True"), 

80 default=['doBrighterFatter', 'doUseOpticsTransmission', 

81 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission'] 

82 ) 

83 isrDesirableSteps = pexConfig.ListField( 

84 dtype=str, 

85 doc=("isr operations that it is advisable to perform, but are not mission-critical." 

86 " WARNs are logged for any of these found to be False."), 

87 default=['doBias'] 

88 ) 

89 ccdKey = pexConfig.Field( 

90 dtype=str, 

91 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'", 

92 default='ccd', 

93 ) 

94 imageTypeKey = pexConfig.Field( 

95 dtype=str, 

96 doc="The key for the butler to use by which to check whether images are darks or flats", 

97 default='imageType', 

98 ) 

99 mode = pexConfig.ChoiceField( 

100 doc=("Use single master calibs (flat and dark) for finding defects, or a list of raw visits?" 

101 " If MASTER, a single visit number should be supplied, for which the corresponding master flat" 

102 " and dark will be used. If VISITS, the list of visits will be used, treating the flats and " 

103 " darks as appropriate, depending on their image types, as determined by their imageType from" 

104 " config.imageTypeKey"), 

105 dtype=str, 

106 default="VISITS", 

107 allowed={ 

108 "VISITS": "Calculate defects from a list of raw visits", 

109 "MASTER": "Use the corresponding master calibs from the specified visit to measure defects", 

110 } 

111 ) 

112 nSigmaBright = pexConfig.Field( 

113 dtype=float, 

114 doc=("Number of sigma above mean for bright pixel detection. The default value was found to be", 

115 " appropriate for some LSST sensors in DM-17490."), 

116 default=4.8, 

117 ) 

118 nSigmaDark = pexConfig.Field( 

119 dtype=float, 

120 doc=("Number of sigma below mean for dark pixel detection. The default value was found to be", 

121 " appropriate for some LSST sensors in DM-17490."), 

122 default=5.0, 

123 ) 

124 nPixBorderUpDown = pexConfig.Field( 

125 dtype=int, 

126 doc="Number of pixels to exclude from top & bottom of image when looking for defects.", 

127 default=7, 

128 ) 

129 nPixBorderLeftRight = pexConfig.Field( 

130 dtype=int, 

131 doc="Number of pixels to exclude from left & right of image when looking for defects.", 

132 default=7, 

133 ) 

134 badOnAndOffPixelColumnThreshold = pexConfig.Field( 

135 dtype=int, 

136 doc=("If BPC is the set of all the bad pixels in a given column (not necessarily consecutive) ", 

137 "and the size of BPC is at least 'badOnAndOffPixelColumnThreshold', all the pixels between the ", 

138 "pixels that satisfy minY (BPC) and maxY (BPC) will be marked as bad, with 'Y' being the long ", 

139 "axis of the amplifier (and 'X' the other axis, which for a column is a constant for all ", 

140 "pixels in the set BPC). If there are more than 'goodPixelColumnGapThreshold' consecutive ", 

141 "non-bad pixels in BPC, an exception to the above is made and those consecutive ", 

142 "'goodPixelColumnGapThreshold' are not marked as bad."), 

143 default=50, 

144 ) 

145 goodPixelColumnGapThreshold = pexConfig.Field( 

146 dtype=int, 

147 doc=("Size, in pixels, of usable consecutive pixels in a column with on and off bad pixels (see ", 

148 "'badOnAndOffPixelColumnThreshold')."), 

149 default=30, 

150 ) 

151 edgesAsDefects = pexConfig.Field( 

152 dtype=bool, 

153 doc=("Mark all edge pixels, as defined by nPixBorder[UpDown, LeftRight], as defects." 

154 " Normal treatment is to simply exclude this region from the defect finding, such that no" 

155 " defect will be located there."), 

156 default=False, 

157 ) 

158 assertSameRun = pexConfig.Field( 

159 dtype=bool, 

160 doc=("Ensure that all visits are from the same run? Raises if this is not the case, or" 

161 "if the run key isn't found."), 

162 default=False, # false because most obs_packages don't have runs. obs_lsst/ts8 overrides this. 

163 ) 

164 ignoreFilters = pexConfig.Field( 

165 dtype=bool, 

166 doc=("Set the filters used in the CALIB_ID to NONE regardless of the filters on the input" 

167 " images. Allows mixing of filters in the input flats. Set to False if you think" 

168 " your defects might be chromatic and want to have registry support for varying" 

169 " defects with respect to filter."), 

170 default=True, 

171 ) 

172 nullFilterName = pexConfig.Field( 

173 dtype=str, 

174 doc=("The name of the null filter if ignoreFilters is True. Usually something like NONE or EMPTY"), 

175 default="NONE", 

176 ) 

177 combinationMode = pexConfig.ChoiceField( 

178 doc="Which types of defects to identify", 

179 dtype=str, 

180 default="FRACTION", 

181 allowed={ 

182 "AND": "Logical AND the pixels found in each visit to form set ", 

183 "OR": "Logical OR the pixels found in each visit to form set ", 

184 "FRACTION": "Use pixels found in more than config.combinationFraction of visits ", 

185 } 

186 ) 

187 combinationFraction = pexConfig.RangeField( 

188 dtype=float, 

189 doc=("The fraction (0..1) of visits in which a pixel was found to be defective across" 

190 " the visit list in order to be marked as a defect. Note, upper bound is exclusive, so use" 

191 " mode AND to require pixel to appear in all images."), 

192 default=0.7, 

193 min=0, 

194 max=1, 

195 ) 

196 makePlots = pexConfig.Field( 

197 dtype=bool, 

198 doc=("Plot histograms for each visit for each amp (one plot per detector) and the final" 

199 " defects overlaid on the sensor."), 

200 default=False, 

201 ) 

202 writeAs = pexConfig.ChoiceField( 

203 doc="Write the output file as ASCII or FITS table", 

204 dtype=str, 

205 default="FITS", 

206 allowed={ 

207 "ASCII": "Write the output as an ASCII file", 

208 "FITS": "Write the output as an FITS table", 

209 "BOTH": "Write the output as both a FITS table and an ASCII file", 

210 } 

211 ) 

212 

213 

214class FindDefectsTask(pipeBase.CmdLineTask): 

215 """Task for finding defects in sensors. 

216 

217 The task has two modes of operation, defect finding in raws and in 

218 master calibrations, which work as follows. 

219 

220 Master calib defect finding 

221 ---------------------------- 

222 

223 A single visit number is supplied, for which the corresponding flat & dark 

224 will be used. This is because, at present at least, there is no way to pass 

225 a calibration exposure ID from the command line to a command line task. 

226 

227 The task retrieves the corresponding dark and flat exposures for the 

228 supplied visit. If a flat is available the task will (be able to) look 

229 for both bright and dark defects. If only a dark is found then only bright 

230 defects will be sought. 

231 

232 All pixels above/below the specified nSigma which lie with the specified 

233 borders for flats/darks are identified as defects. 

234 

235 Raw visit defect finding 

236 ------------------------ 

237 

238 A list of exposure IDs are supplied for defect finding. The task will 

239 detect bright pixels in the dark frames, if supplied, and bright & dark 

240 pixels in the flats, if supplied, i.e. if you only supply darks you will 

241 only be given bright defects. This is done automatically from the imageType 

242 of the exposure, so the input exposure list can be a mix. 

243 

244 As with the master calib detection, all pixels above/below the specified 

245 nSigma which lie with the specified borders for flats/darks are identified 

246 as defects. Then, a post-processing step is done to merge these detections, 

247 with pixels appearing in a fraction [0..1] of the images are kept as defects 

248 and those appearing below that occurrence-threshold are discarded. 

249 """ 

250 

251 RunnerClass = SingleVisitListTaskRunner 

252 ConfigClass = FindDefectsTaskConfig 

253 _DefaultName = "findDefects" 

254 

255 def __init__(self, *args, **kwargs): 

256 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

257 self.makeSubtask("isrForFlats") 

258 self.makeSubtask("isrForDarks") 

259 

260 validateIsrConfig(self.isrForFlats, self.config.isrMandatoryStepsFlats, 

261 self.config.isrForbiddenStepsFlats, self.config.isrDesirableSteps) 

262 validateIsrConfig(self.isrForDarks, self.config.isrMandatoryStepsDarks, 

263 self.config.isrForbiddenStepsDarks, self.config.isrDesirableSteps) 

264 self.config.validate() 

265 self.config.freeze() 

266 

267 @classmethod 

268 def _makeArgumentParser(cls): 

269 """Augment argument parser for the FindDefectsTask.""" 

270 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

271 parser.add_argument("--visitList", dest="visitList", nargs="*", 

272 help=("List of visits to use. Same for each detector." 

273 " Uses the normal 0..10:3^234 syntax")) 

274 parser.add_id_argument("--id", datasetType="newDefects", 

275 ContainerClass=NonexistentDatasetTaskDataIdContainer, 

276 help="The ccds to use, e.g. --id ccd=0..100") 

277 return parser 

278 

279 @pipeBase.timeMethod 

280 def runDataRef(self, dataRef, visitList): 

281 """Run the defect finding task. 

282 

283 Find the defects, as described in the main task docstring, from a 

284 dataRef and a list of visit(s). 

285 

286 Parameters 

287 ---------- 

288 dataRef : `lsst.daf.persistence.ButlerDataRef` 

289 dataRef for the detector for the visits to be fit. 

290 visitList : `list` [`int`] 

291 List of visits to be processed. If config.mode == 'VISITS' then the 

292 list of visits is used. If config.mode == 'MASTER' then the length 

293 of visitList must be one, and the corresponding master calibrations 

294 are used. 

295 

296 Returns 

297 ------- 

298 result : `lsst.pipe.base.Struct` 

299 Result struct with Components: 

300 

301 - ``defects`` : `lsst.meas.algorithms.Defect` 

302 The defects found by the task. 

303 - ``exitStatus`` : `int` 

304 The exit code. 

305 """ 

306 

307 detNum = dataRef.dataId[self.config.ccdKey] 

308 self.log.info("Calculating defects using %s visits for detector %s" % (visitList, detNum)) 

309 

310 defectLists = {'dark': [], 'flat': []} 

311 

312 midTime = 0 

313 filters = set() 

314 

315 if self.config.mode == 'MASTER': 

316 if len(visitList) > 1: 

317 raise RuntimeError(f"Must only specify one visit when using mode MASTER, got {visitList}") 

318 dataRef.dataId['expId'] = visitList[0] 

319 

320 for datasetType in defectLists.keys(): 

321 exp = dataRef.get(datasetType) 

322 midTime += self._getMjd(exp) 

323 filters.add(exp.getFilter().getName()) 

324 defects = self.findHotAndColdPixels(exp, datasetType) 

325 

326 msg = "Found %s defects containing %s pixels in master %s" 

327 self.log.info(msg, len(defects), self._nPixFromDefects(defects), datasetType) 

328 defectLists[datasetType].append(defects) 

329 if self.config.makePlots: 

330 self._plot(dataRef, exp, visitList[0], self._getNsigmaForPlot(datasetType), 

331 defects, datasetType) 

332 midTime /= len(defectLists.keys()) 

333 

334 elif self.config.mode == 'VISITS': 

335 butler = dataRef.getButler() 

336 

337 if self.config.assertSameRun: 

338 runs = self._getRunListFromVisits(butler, visitList) 

339 if len(runs) != 1: 

340 raise RuntimeError(f"Got data from runs {runs} with assertSameRun==True") 

341 

342 for visit in visitList: 

343 imageType = butler.queryMetadata('raw', self.config.imageTypeKey, dataId={'expId': visit})[0] 

344 imageType = imageType.lower() 

345 dataRef.dataId['expId'] = visit 

346 

347 if imageType == 'flat': # note different isr tasks 

348 exp = self.isrForFlats.runDataRef(dataRef).exposure 

349 defects = self.findHotAndColdPixels(exp, imageType) 

350 defectLists['flat'].append(defects) 

351 midTime += self._getMjd(exp) 

352 filters.add(exp.getFilter().getName()) 

353 

354 elif imageType == 'dark': 

355 exp = self.isrForDarks.runDataRef(dataRef).exposure 

356 defects = self.findHotAndColdPixels(exp, imageType) 

357 defectLists['dark'].append(defects) 

358 midTime += self._getMjd(exp) 

359 filters.add(exp.getFilter().getName()) 

360 

361 else: 

362 raise RuntimeError(f"Failed on imageType {imageType}. Only flats and darks supported") 

363 

364 msg = "Found %s defects containing %s pixels in visit %s" 

365 self.log.info(msg, len(defects), self._nPixFromDefects(defects), visit) 

366 

367 if self.config.makePlots: 

368 self._plot(dataRef, exp, visit, self._getNsigmaForPlot(imageType), defects, imageType) 

369 

370 midTime /= len(visitList) 

371 

372 msg = "Combining %s defect sets from darks for detector %s" 

373 self.log.info(msg, len(defectLists['dark']), detNum) 

374 mergedDefectsFromDarks = self._postProcessDefectSets(defectLists['dark'], exp.getDimensions(), 

375 self.config.combinationMode) 

376 msg = "Combining %s defect sets from flats for detector %s" 

377 self.log.info(msg, len(defectLists['flat']), detNum) 

378 mergedDefectsFromFlats = self._postProcessDefectSets(defectLists['flat'], exp.getDimensions(), 

379 self.config.combinationMode) 

380 

381 msg = "Combining bright and dark defect sets for detector %s" 

382 self.log.info(msg, detNum) 

383 brightDarkPostMerge = [mergedDefectsFromDarks, mergedDefectsFromFlats] 

384 allDefects = self._postProcessDefectSets(brightDarkPostMerge, exp.getDimensions(), mode='OR') 

385 

386 self._writeData(dataRef, allDefects, midTime, filters) 

387 

388 self.log.info("Finished finding defects in detector %s" % detNum) 

389 return pipeBase.Struct(defects=allDefects, exitStatus=0) 

390 

391 def _getNsigmaForPlot(self, imageType): 

392 assert imageType in ['flat', 'dark'] 

393 nSig = self.config.nSigmaBright if imageType == 'flat' else self.config.nSigmaDark 

394 return nSig 

395 

396 @staticmethod 

397 def _nPixFromDefects(defect): 

398 """Count the number of pixels in a defect object.""" 

399 nPix = 0 

400 for d in defect: 

401 nPix += d.getBBox().getArea() 

402 return nPix 

403 

404 def _writeData(self, dataRef, defects, midTime, filters): 

405 """Write the data out to the defect file. 

406 

407 Parameters 

408 ---------- 

409 dataRef : `lsst.daf.persistence.ButlerDataRef` 

410 dataRef for the detector for defects to be written. 

411 defects : `lsst.meas.algorithms.Defect` 

412 The defects to be written. 

413 """ 

414 date = dafBase.DateTime(midTime, dafBase.DateTime.MJD).toPython().isoformat() 

415 

416 detName = self._getDetectorNameShort(dataRef) 

417 instrumentName = self._getInstrumentName(dataRef) 

418 detNum = self._getDetectorNumber(dataRef) 

419 if not self.config.ignoreFilters: 

420 filt = self._filterSetToFilterString(filters) 

421 else: 

422 filt = self.config.nullFilterName 

423 

424 CALIB_ID = f"detectorName={detName} detector={detNum} calibDate={date} ccd={detNum} filter={filt}" 

425 try: 

426 raftName = self._getRaftName(dataRef) 

427 CALIB_ID += f" raftName={raftName}" 

428 except Exception: 

429 pass 

430 

431 now = dafBase.DateTime.now().toPython() 

432 mdOriginal = defects.getMetadata() 

433 mdSupplemental = {"INSTRUME": instrumentName, 

434 "DETECTOR": dataRef.dataId['detector'], 

435 "CALIBDATE": date, 

436 "CALIB_ID": CALIB_ID, 

437 "CALIB_CREATION_DATE": now.date().isoformat(), 

438 "CALIB_CREATION_TIME": now.time().isoformat()} 

439 

440 mdOriginal.update(mdSupplemental) 

441 

442 # TODO: DM-23508 sort out the butler abuse from here-on down in Gen3 

443 # defects should simply be butler.put() 

444 templateFilename = dataRef.getUri(write=True) # does not guarantee that full path exists 

445 baseDirName = os.path.dirname(templateFilename) 

446 # ingest curated calibs demands detectorName is lowercase 

447 detNameFull = self._getDetectorNameFull(dataRef) 

448 dirName = os.path.join(baseDirName, instrumentName, "defects", detNameFull.lower()) 

449 if not os.path.exists(dirName): 

450 os.makedirs(dirName) 

451 

452 date += ".fits" 

453 filename = os.path.join(dirName, date) 

454 

455 msg = "Writing defects to %s in format: %s" 

456 self.log.info(msg, os.path.splitext(filename)[0], self.config.writeAs) 

457 if self.config.writeAs in ['FITS', 'BOTH']: 

458 defects.writeFits(filename) 

459 if self.config.writeAs in ['ASCII', 'BOTH']: 

460 wroteTo = defects.writeText(filename) 

461 assert(os.path.splitext(wroteTo)[0] == os.path.splitext(filename)[0]) 

462 return 

463 

464 @staticmethod 

465 def _filterSetToFilterString(filters): 

466 return "~".join([f for f in filters]) 

467 

468 @staticmethod 

469 def _getDetectorNumber(dataRef): 

470 """The detector's integer identifier.""" 

471 dataRefDetNum = dataRef.dataId['detector'] 

472 camera = dataRef.get('camera') 

473 detectorDetNum = camera[dataRef.dataId['detector']].getId() 

474 assert dataRefDetNum == detectorDetNum 

475 return dataRefDetNum 

476 

477 @staticmethod 

478 def _getInstrumentName(dataRef): 

479 camera = dataRef.get('camera') 

480 return camera.getName() 

481 

482 @staticmethod 

483 def _getDetectorNameFull(dataRef): 

484 """The detector's self-reported full name, e.g. R12_S01.""" 

485 camera = dataRef.get('camera') 

486 return camera[dataRef.dataId['detector']].getName() 

487 

488 @staticmethod 

489 def _getDetectorNameShort(dataRef): 

490 """The detectorName per the butler, e.g. slot name, e.g. S12.""" 

491 butler = dataRef.getButler() 

492 detectorName = butler.queryMetadata('raw', ['detectorName'], dataRef.dataId)[0] 

493 return detectorName 

494 

495 @staticmethod 

496 def _getRaftName(dataRef): 

497 """The detectorName per the butler, e.g. slot name, e.g. S12.""" 

498 butler = dataRef.getButler() 

499 raftName = butler.queryMetadata('raw', ['raftName'], dataRef.dataId)[0] 

500 return raftName 

501 

502 @staticmethod 

503 def _getMjd(exp, timescale=dafBase.DateTime.UTC): 

504 vi = exp.getInfo().getVisitInfo() 

505 dateObs = vi.getDate() 

506 mjd = dateObs.get(dafBase.DateTime.MJD) 

507 return mjd 

508 

509 @staticmethod 

510 def _getRunListFromVisits(butler, visitList): 

511 """Return the set of runs for the visits in visitList.""" 

512 runs = set() 

513 for visit in visitList: 

514 runs.add(butler.queryMetadata('raw', 'run', dataId={'expId': visit})[0]) 

515 return runs 

516 

517 def _postProcessDefectSets(self, defectList, imageDimensions, mode): 

518 """Combine a list of defects to make a single defect object. 

519 

520 AND, OR or use percentage of visits in which defects appear 

521 depending on config. 

522 

523 Parameters 

524 ---------- 

525 defectList : `list` [`lsst.meas.algorithms.Defect`] 

526 The lList of defects to merge. 

527 imageDimensions : `tuple` [`int`] 

528 The size of the image. 

529 mode : `str` 

530 The combination mode to use, either 'AND', 'OR' or 'FRACTION' 

531 

532 Returns 

533 ------- 

534 defects : `lsst.meas.algorithms.Defect` 

535 The defect set resulting from the merge. 

536 """ 

537 # so that empty lists can be passed in for input data 

538 # where only flats or darks are supplied 

539 if defectList == []: 

540 return [] 

541 

542 if len(defectList) == 1: # single input - no merging to do 

543 return defectList[0] 

544 

545 sumImage = afwImage.MaskedImageF(imageDimensions) 

546 for defects in defectList: 

547 for defect in defects: 

548 sumImage.image[defect.getBBox()] += 1 

549 sumImage /= len(defectList) 

550 

551 nDetected = len(np.where(sumImage.image.array > 0)[0]) 

552 self.log.info("Pre-merge %s pixels with non-zero detections" % nDetected) 

553 

554 if mode == 'OR': # must appear in any 

555 indices = np.where(sumImage.image.array > 0) 

556 else: 

557 if mode == 'AND': # must appear in all 

558 threshold = 1 

559 elif mode == 'FRACTION': 

560 threshold = self.config.combinationFraction 

561 else: 

562 raise RuntimeError(f"Got unsupported combinationMode {mode}") 

563 indices = np.where(sumImage.image.array >= threshold) 

564 

565 BADBIT = sumImage.mask.getPlaneBitMask('BAD') 

566 sumImage.mask.array[indices] |= BADBIT 

567 

568 self.log.info("Post-merge %s pixels marked as defects" % len(indices[0])) 

569 

570 if self.config.edgesAsDefects: 

571 self.log.info("Masking edge pixels as defects in addition to previously identified defects") 

572 self._setEdgeBits(sumImage, 'BAD') 

573 

574 defects = measAlg.Defects.fromMask(sumImage, 'BAD') 

575 return defects 

576 

577 @staticmethod 

578 def _getNumGoodPixels(maskedIm, badMaskString="NO_DATA"): 

579 """Return the number of non-bad pixels in the image.""" 

580 nPixels = maskedIm.mask.array.size 

581 nBad = countMaskedPixels(maskedIm, badMaskString) 

582 return nPixels - nBad 

583 

584 def findHotAndColdPixels(self, exp, imageType, setMask=False): 

585 """Find hot and cold pixels in an image. 

586 

587 Using config-defined thresholds on a per-amp basis, mask pixels 

588 that are nSigma above threshold in dark frames (hot pixels), 

589 or nSigma away from the clipped mean in flats (hot & cold pixels). 

590 

591 Parameters 

592 ---------- 

593 exp : `lsst.afw.image.exposure.Exposure` 

594 The exposure in which to find defects. 

595 imageType : `str` 

596 The image type, either 'dark' or 'flat'. 

597 setMask : `bool` 

598 If true, update exp with hot and cold pixels. 

599 hot: DETECTED 

600 cold: DETECTED_NEGATIVE 

601 

602 Returns 

603 ------- 

604 defects : `lsst.meas.algorithms.Defect` 

605 The defects found in the image. 

606 """ 

607 assert imageType in ['flat', 'dark'] 

608 

609 self._setEdgeBits(exp) 

610 maskedIm = exp.maskedImage 

611 

612 # the detection polarity for afwDetection, True for positive, 

613 # False for negative, and therefore True for darks as they only have 

614 # bright pixels, and both for flats, as they have bright and dark pix 

615 polarities = {'dark': [True], 'flat': [True, False]}[imageType] 

616 

617 footprintList = [] 

618 

619 for amp in exp.getDetector(): 

620 ampImg = maskedIm[amp.getBBox()].clone() 

621 

622 # crop ampImage depending on where the amp lies in the image 

623 if self.config.nPixBorderLeftRight: 

624 if ampImg.getX0() == 0: 

625 ampImg = ampImg[self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] 

626 else: 

627 ampImg = ampImg[:-self.config.nPixBorderLeftRight, :, afwImage.LOCAL] 

628 if self.config.nPixBorderUpDown: 

629 if ampImg.getY0() == 0: 

630 ampImg = ampImg[:, self.config.nPixBorderUpDown:, afwImage.LOCAL] 

631 else: 

632 ampImg = ampImg[:, :-self.config.nPixBorderUpDown, afwImage.LOCAL] 

633 

634 if self._getNumGoodPixels(ampImg) == 0: # amp contains no usable pixels 

635 continue 

636 

637 ampImg -= afwMath.makeStatistics(ampImg, afwMath.MEANCLIP, ).getValue() 

638 

639 mergedSet = None 

640 for polarity in polarities: 

641 nSig = self.config.nSigmaBright if polarity else self.config.nSigmaDark 

642 threshold = afwDetection.createThreshold(nSig, 'stdev', polarity=polarity) 

643 

644 footprintSet = afwDetection.FootprintSet(ampImg, threshold) 

645 if setMask: 

646 footprintSet.setMask(maskedIm.mask, ("DETECTED" if polarity else "DETECTED_NEGATIVE")) 

647 

648 if mergedSet is None: 

649 mergedSet = footprintSet 

650 else: 

651 mergedSet.merge(footprintSet) 

652 

653 footprintList += mergedSet.getFootprints() 

654 

655 defects = measAlg.Defects.fromFootprintList(footprintList) 

656 defects = self.maskBlocksIfIntermitentBadPixelsInColumn(defects) 

657 

658 return defects 

659 

660 def maskBlocksIfIntermitentBadPixelsInColumn(self, defects): 

661 """Mask blocks in a column if there are on-and-off bad pixels 

662 

663 If there's a column with on and off bad pixels, mask all the pixels in between, 

664 except if there is a large enough gap of consecutive good pixels between two 

665 bad pixels in the column. 

666 

667 Parameters 

668 --------- 

669 defects: `lsst.meas.algorithms.Defect` 

670 The defects found in the image so far 

671 

672 Returns 

673 ------ 

674 defects: `lsst.meas.algorithms.Defect` 

675 If the number of bad pixels in a column is not larger or equal than 

676 self.config.badPixelColumnThreshold, the iput list is returned. Otherwise, 

677 the defects list returned will include boxes that mask blocks of on-and-of 

678 pixels. 

679 """ 

680 # Get the (x, y) values of each bad pixel in amp. 

681 coordinates = [] 

682 for defect in defects: 

683 bbox = defect.getBBox() 

684 x0, y0 = bbox.getMinX(), bbox.getMinY() 

685 deltaX0, deltaY0 = bbox.getDimensions() 

686 for j in np.arange(y0, y0+deltaY0): 

687 for i in np.arange(x0, x0 + deltaX0): 

688 coordinates.append((i, j)) 

689 

690 x, y = [], [] 

691 for coordinatePair in coordinates: 

692 x.append(coordinatePair[0]) 

693 y.append(coordinatePair[1]) 

694 

695 x = np.array(x) 

696 y = np.array(y) 

697 # Find the defects with same "x" (vertical) coordinate (column). 

698 unique, counts = np.unique(x, return_counts=True) 

699 multipleX = [] 

700 for (a, b) in zip(unique, counts): 

701 if b >= self.config.badOnAndOffPixelColumnThreshold: 

702 multipleX.append(a) 

703 if len(multipleX) != 0: 

704 defects = self._markBlocksInBadColumn(x, y, multipleX, defects) 

705 

706 return defects 

707 

708 def _markBlocksInBadColumn(self, x, y, multipleX, defects): 

709 """Mask blocks in a column if number of on-and-off bad pixels is above threshold. 

710 

711 This function is called if the number of on-and-off bad pixels in a column 

712 is larger or equal than self.config.badOnAndOffPixelColumnThreshold. 

713 

714 Parameters 

715 --------- 

716 x: list 

717 Lower left x coordinate of defect box. x coordinate is along the short axis if amp. 

718 

719 y: list 

720 Lower left y coordinate of defect box. x coordinate is along the long axis if amp. 

721 

722 multipleX: list 

723 List of x coordinates in amp. with multiple bad pixels (i.e., columns with defects). 

724 

725 defects: `lsst.meas.algorithms.Defect` 

726 The defcts found in the image so far 

727 

728 Returns 

729 ------- 

730 defects: `lsst.meas.algorithms.Defect` 

731 The defects list returned that will include boxes that mask blocks 

732 of on-and-of pixels. 

733 """ 

734 with defects.bulk_update(): 

735 goodPixelColumnGapThreshold = self.config.goodPixelColumnGapThreshold 

736 for x0 in multipleX: 

737 index = np.where(x == x0) 

738 multipleY = y[index] # multipleY and multipleX are in 1-1 correspondence. 

739 minY, maxY = np.min(multipleY), np.max(multipleY) 

740 # Next few lines: don't mask pixels in column if gap of good pixels between 

741 # two consecutive bad pixels is larger or equal than 'goodPixelColumnGapThreshold'. 

742 diffIndex = np.where(np.diff(multipleY) >= goodPixelColumnGapThreshold)[0] 

743 if len(diffIndex) != 0: 

744 limits = [minY] # put the minimum first 

745 for gapIndex in diffIndex: 

746 limits.append(multipleY[gapIndex]) 

747 limits.append(multipleY[gapIndex+1]) 

748 limits.append(maxY) # maximum last 

749 assert len(limits)%2 == 0, 'limits is even by design, but check anyways' 

750 for i in np.arange(0, len(limits)-1, 2): 

751 s = Box2I(minimum=Point2I(x0, limits[i]), maximum=Point2I(x0, limits[i+1])) 

752 defects.append(s) 

753 else: # No gap is large enough 

754 s = Box2I(minimum=Point2I(x0, minY), maximum=Point2I(x0, maxY)) 

755 defects.append(s) 

756 return defects 

757 

758 def _setEdgeBits(self, exposureOrMaskedImage, maskplaneToSet='EDGE'): 

759 """Set edge bits on an exposure or maskedImage. 

760 

761 Raises 

762 ------ 

763 TypeError 

764 Raised if parameter ``exposureOrMaskedImage`` is an invalid type. 

765 """ 

766 if isinstance(exposureOrMaskedImage, afwImage.Exposure): 

767 mi = exposureOrMaskedImage.maskedImage 

768 elif isinstance(exposureOrMaskedImage, afwImage.MaskedImage): 

769 mi = exposureOrMaskedImage 

770 else: 

771 t = type(exposureOrMaskedImage) 

772 raise TypeError(f"Function supports exposure or maskedImage but not {t}") 

773 

774 MASKBIT = mi.mask.getPlaneBitMask(maskplaneToSet) 

775 if self.config.nPixBorderLeftRight: 

776 mi.mask[: self.config.nPixBorderLeftRight, :, afwImage.LOCAL] |= MASKBIT 

777 mi.mask[-self.config.nPixBorderLeftRight:, :, afwImage.LOCAL] |= MASKBIT 

778 if self.config.nPixBorderUpDown: 

779 mi.mask[:, : self.config.nPixBorderUpDown, afwImage.LOCAL] |= MASKBIT 

780 mi.mask[:, -self.config.nPixBorderUpDown:, afwImage.LOCAL] |= MASKBIT 

781 

782 def _plot(self, dataRef, exp, visit, nSig, defects, imageType): # pragma: no cover 

783 """Plot the defects and pixel histograms. 

784 

785 Parameters 

786 ---------- 

787 dataRef : `lsst.daf.persistence.ButlerDataRef` 

788 dataRef for the detector. 

789 exp : `lsst.afw.image.exposure.Exposure` 

790 The exposure in which the defects were found. 

791 visit : `int` 

792 The visit number. 

793 nSig : `float` 

794 The number of sigma used for detection 

795 defects : `lsst.meas.algorithms.Defect` 

796 The defects to plot. 

797 imageType : `str` 

798 The type of image, either 'dark' or 'flat'. 

799 

800 Currently only for LSST sensors. Plots are written to the path 

801 given by the butler for the ``cpPipePlotRoot`` dataset type. 

802 """ 

803 import matplotlib.pyplot as plt 

804 from matplotlib.backends.backend_pdf import PdfPages 

805 

806 afwDisplay.setDefaultBackend("matplotlib") 

807 plt.interactive(False) # seems to need reasserting here 

808 

809 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True) 

810 if not os.path.exists(dirname): 

811 os.makedirs(dirname) 

812 

813 detNum = exp.getDetector().getId() 

814 nAmps = len(exp.getDetector()) 

815 

816 if self.config.mode == "MASTER": 

817 filename = f"defectPlot_det{detNum}_master-{imageType}_for-exp{visit}.pdf" 

818 elif self.config.mode == "VISITS": 

819 filename = f"defectPlot_det{detNum}_{imageType}_exp{visit}.pdf" 

820 

821 filenameFull = os.path.join(dirname, filename) 

822 

823 with warnings.catch_warnings(): 

824 msg = "Matplotlib is currently using agg, which is a non-GUI backend, so cannot show the figure." 

825 warnings.filterwarnings("ignore", message=msg) 

826 with PdfPages(filenameFull) as pdfPages: 

827 if nAmps == 16: 

828 self._plotAmpHistogram(dataRef, exp, visit, nSig) 

829 pdfPages.savefig() 

830 

831 self._plotDefects(exp, visit, defects, imageType) 

832 pdfPages.savefig() 

833 self.log.info("Wrote plot(s) to %s" % filenameFull) 

834 

835 def _plotDefects(self, exp, visit, defects, imageType): # pragma: no cover 

836 """Plot the defects found by the task. 

837 

838 Parameters 

839 ---------- 

840 exp : `lsst.afw.image.exposure.Exposure` 

841 The exposure in which the defects were found. 

842 visit : `int` 

843 The visit number. 

844 defects : `lsst.meas.algorithms.Defect` 

845 The defects to plot. 

846 imageType : `str` 

847 The type of image, either 'dark' or 'flat'. 

848 """ 

849 expCopy = exp.clone() # we mess with the copy later, so make a clone 

850 del exp # del for safety - no longer needed as we have a copy so remove from scope to save mistakes 

851 maskedIm = expCopy.maskedImage 

852 

853 defects.maskPixels(expCopy.maskedImage, "BAD") 

854 detector = expCopy.getDetector() 

855 

856 disp = afwDisplay.Display(0, reopenPlot=True, dpi=200) 

857 

858 if imageType == "flat": # set each amp image to have a mean of 1.00 

859 for amp in detector: 

860 ampIm = maskedIm.image[amp.getBBox()] 

861 ampIm -= afwMath.makeStatistics(ampIm, afwMath.MEANCLIP).getValue() + 1 

862 

863 mpDict = maskedIm.mask.getMaskPlaneDict() 

864 for plane in mpDict.keys(): 

865 if plane in ['BAD']: 

866 continue 

867 disp.setMaskPlaneColor(plane, afwDisplay.IGNORE) 

868 

869 disp.scale('asinh', 'zscale') 

870 disp.setMaskTransparency(80) 

871 disp.setMaskPlaneColor("BAD", afwDisplay.RED) 

872 

873 disp.setImageColormap('gray') 

874 title = (f"Detector: {detector.getName()[-3:]} {detector.getSerial()}" 

875 f", Type: {imageType}, visit: {visit}") 

876 disp.mtv(maskedIm, title=title) 

877 

878 cameraGeom.utils.overlayCcdBoxes(detector, isTrimmed=True, display=disp) 

879 

880 def _plotAmpHistogram(self, dataRef, exp, visit, nSigmaUsed): # pragma: no cover 

881 """ 

882 Make a histogram of the distribution of pixel values for each amp. 

883 

884 The main image data histogram is plotted in blue. Edge pixels, 

885 if masked, are in red. Note that masked edge pixels do not contribute 

886 to the underflow and overflow numbers. 

887 

888 Note that this currently only supports the 16-amp LSST detectors. 

889 

890 Parameters 

891 ---------- 

892 dataRef : `lsst.daf.persistence.ButlerDataRef` 

893 dataRef for the detector. 

894 exp : `lsst.afw.image.exposure.Exposure` 

895 The exposure in which the defects were found. 

896 visit : `int` 

897 The visit number. 

898 nSigmaUsed : `float` 

899 The number of sigma used for detection 

900 """ 

901 import matplotlib.pyplot as plt 

902 

903 detector = exp.getDetector() 

904 

905 if len(detector) != 16: 

906 raise RuntimeError("Plotting currently only supported for 16 amp detectors") 

907 fig, ax = plt.subplots(nrows=4, ncols=4, sharex='col', sharey='row', figsize=(13, 10)) 

908 

909 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

910 

911 for (amp, a) in zip(reversed(detector), ax.flatten()): 

912 mi = exp.maskedImage[amp.getBBox()] 

913 

914 # normalize by expTime as we plot in ADU/s and don't always work with master calibs 

915 mi.image.array /= expTime 

916 stats = afwMath.makeStatistics(mi, afwMath.MEANCLIP | afwMath.STDEVCLIP) 

917 mean, sigma = stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP) 

918 

919 # Get array of pixels 

920 EDGEBIT = exp.maskedImage.mask.getPlaneBitMask("EDGE") 

921 imgData = mi.image.array[(mi.mask.array & EDGEBIT) == 0].flatten() 

922 edgeData = mi.image.array[(mi.mask.array & EDGEBIT) != 0].flatten() 

923 

924 thrUpper = mean + nSigmaUsed*sigma 

925 thrLower = mean - nSigmaUsed*sigma 

926 

927 nRight = len(imgData[imgData > thrUpper]) 

928 nLeft = len(imgData[imgData < thrLower]) 

929 

930 nsig = nSigmaUsed + 1.2 # add something small so the edge of the plot is out from level used 

931 leftEdge = mean - nsig * nSigmaUsed*sigma 

932 rightEdge = mean + nsig * nSigmaUsed*sigma 

933 nbins = np.linspace(leftEdge, rightEdge, 1000) 

934 ey, bin_borders, patches = a.hist(edgeData, histtype='step', bins=nbins, lw=1, edgecolor='red') 

935 y, bin_borders, patches = a.hist(imgData, histtype='step', bins=nbins, lw=3, edgecolor='blue') 

936 

937 # Report number of entries in over-and -underflow bins, i.e. off the edges of the histogram 

938 nOverflow = len(imgData[imgData > rightEdge]) 

939 nUnderflow = len(imgData[imgData < leftEdge]) 

940 

941 # Put v-lines and textboxes in 

942 a.axvline(thrUpper, c='k') 

943 a.axvline(thrLower, c='k') 

944 msg = f"{amp.getName()}\nmean:{mean: .2f}\n$\\sigma$:{sigma: .2f}" 

945 a.text(0.65, 0.6, msg, transform=a.transAxes, fontsize=11) 

946 msg = f"nLeft:{nLeft}\nnRight:{nRight}\nnOverflow:{nOverflow}\nnUnderflow:{nUnderflow}" 

947 a.text(0.03, 0.6, msg, transform=a.transAxes, fontsize=11.5) 

948 

949 # set axis limits and scales 

950 a.set_ylim([1., 1.7*np.max(y)]) 

951 lPlot, rPlot = a.get_xlim() 

952 a.set_xlim(np.array([lPlot, rPlot])) 

953 a.set_yscale('log') 

954 a.set_xlabel("ADU/s") 

955 

956 return