Coverage for python/lsst/ip/isr/defects.py: 19%

269 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-10 11:33 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Support for image defects""" 

22 

23__all__ = ("Defects",) 

24 

25import logging 

26import itertools 

27import contextlib 

28import numpy as np 

29import math 

30import numbers 

31import astropy.table 

32 

33import lsst.geom 

34import lsst.afw.table 

35import lsst.afw.detection 

36import lsst.afw.image 

37import lsst.afw.geom 

38from lsst.meas.algorithms import Defect 

39from .calibType import IsrCalib 

40 

41log = logging.getLogger(__name__) 

42 

43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA" 

44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION" 

45 

46 

47class Defects(IsrCalib): 

48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`. 

49 

50 Parameters 

51 ---------- 

52 defectList : iterable, optional 

53 Collections of defects to apply to the image. Can be an iterable of 

54 `lsst.meas.algorithms.Defect` or `lsst.geom.BoxI`. 

55 metadata : `lsst.daf.base.PropertyList`, optional 

56 Metadata to associate with the defects. Will be copied and 

57 overwrite existing metadata, if any. If not supplied the existing 

58 metadata will be reset. 

59 normalize_on_init : `bool` 

60 If True, normalization is applied to the defects in ``defectList`` to 

61 remove duplicates, eliminate overlaps, etc. 

62 

63 Notes 

64 ----- 

65 Defects are stored within this collection in a "reduced" or "normalized" 

66 form: rather than simply storing the bounding boxes which are added to the 

67 collection, we eliminate overlaps and duplicates. This normalization 

68 procedure may introduce overhead when adding many new defects; it may be 

69 temporarily disabled using the `Defects.bulk_update` context manager if 

70 necessary. 

71 

72 The attributes stored in this calibration are: 

73 

74 _defects : `list` [`lsst.meas.algorithms.Defect`] 

75 The collection of Defect objects. 

76 """ 

77 

78 """The calibration type used for ingest.""" 

79 _OBSTYPE = "defects" 

80 _SCHEMA = '' 

81 _VERSION = 2.0 

82 

83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs): 

84 self._defects = [] 

85 

86 if defectList is not None: 

87 self._bulk_update = True 

88 for d in defectList: 

89 self.append(d) 

90 self._bulk_update = False 

91 

92 if normalize_on_init: 

93 self._normalize() 

94 

95 super().__init__(**kwargs) 

96 self.requiredAttributes.update(['_defects']) 

97 

98 def _check_value(self, value): 

99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect` 

100 or can be converted to one. 

101 

102 Parameters 

103 ---------- 

104 value : `object` 

105 Value to check. 

106 

107 Returns 

108 ------- 

109 new : `~lsst.meas.algorithms.Defect` 

110 Either the supplied value or a new object derived from it. 

111 

112 Raises 

113 ------ 

114 ValueError 

115 Raised if the supplied value can not be converted to 

116 `~lsst.meas.algorithms.Defect` 

117 """ 

118 if isinstance(value, Defect): 

119 pass 

120 elif isinstance(value, lsst.geom.BoxI): 

121 value = Defect(value) 

122 elif isinstance(value, lsst.geom.PointI): 

123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1))) 

124 elif isinstance(value, lsst.afw.image.DefectBase): 

125 value = Defect(value.getBBox()) 

126 else: 

127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'") 

128 return value 

129 

130 def __len__(self): 

131 return len(self._defects) 

132 

133 def __getitem__(self, index): 

134 return self._defects[index] 

135 

136 def __setitem__(self, index, value): 

137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI` 

138 """ 

139 self._defects[index] = self._check_value(value) 

140 self._normalize() 

141 

142 def __iter__(self): 

143 return iter(self._defects) 

144 

145 def __delitem__(self, index): 

146 del self._defects[index] 

147 

148 def __eq__(self, other): 

149 """Compare if two `Defects` are equal. 

150 

151 Two `Defects` are equal if their bounding boxes are equal and in 

152 the same order. Metadata content is ignored. 

153 """ 

154 super().__eq__(other) 

155 

156 if not isinstance(other, self.__class__): 

157 return False 

158 

159 # checking the bboxes with zip() only works if same length 

160 if len(self) != len(other): 

161 return False 

162 

163 # Assume equal if bounding boxes are equal 

164 for d1, d2 in zip(self, other): 

165 if d1.getBBox() != d2.getBBox(): 

166 return False 

167 

168 return True 

169 

170 def __str__(self): 

171 baseStr = super().__str__() 

172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")" 

173 

174 def _normalize(self): 

175 """Recalculate defect bounding boxes for efficiency. 

176 

177 Notes 

178 ----- 

179 Ideally, this would generate the provably-minimal set of bounding 

180 boxes necessary to represent the defects. At present, however, that 

181 doesn't happen: see DM-24781. In the cases of substantial overlaps or 

182 duplication, though, this will produce a much reduced set. 

183 """ 

184 # In bulk-update mode, normalization is a no-op. 

185 if self._bulk_update: 

186 return 

187 

188 # If we have no defects, there is nothing to normalize. 

189 if len(self) == 0: 

190 return 

191 

192 # work out the minimum and maximum bounds from all defect regions. 

193 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf') 

194 for defect in self: 

195 bbox = defect.getBBox() 

196 minX = min(minX, bbox.getMinX()) 

197 minY = min(minY, bbox.getMinY()) 

198 maxX = max(maxX, bbox.getMaxX()) 

199 maxY = max(maxY, bbox.getMaxY()) 

200 

201 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY), 

202 lsst.geom.Point2I(maxX, maxY)) 

203 

204 mask = lsst.afw.image.Mask(region) 

205 self.maskPixels(mask, maskName="BAD") 

206 self._defects = Defects.fromMask(mask, "BAD")._defects 

207 

208 @contextlib.contextmanager 

209 def bulk_update(self): 

210 """Temporarily suspend normalization of the defect list. 

211 """ 

212 self._bulk_update = True 

213 try: 

214 yield 

215 finally: 

216 self._bulk_update = False 

217 self._normalize() 

218 

219 def append(self, value): 

220 self._defects.append(self._check_value(value)) 

221 self._normalize() 

222 

223 def insert(self, index, value): 

224 self._defects.insert(index, self._check_value(value)) 

225 self._normalize() 

226 

227 def copy(self): 

228 """Copy the defects to a new list, creating new defects from the 

229 bounding boxes. 

230 

231 Returns 

232 ------- 

233 new : `Defects` 

234 New list with new `Defect` entries. 

235 

236 Notes 

237 ----- 

238 This is not a shallow copy in that new `Defect` instances are 

239 created from the original bounding boxes. It's also not a deep 

240 copy since the bounding boxes are not recreated. 

241 """ 

242 return self.__class__(d.getBBox() for d in self) 

243 

244 def transpose(self): 

245 """Make a transposed copy of this defect list. 

246 

247 Returns 

248 ------- 

249 retDefectList : `Defects` 

250 Transposed list of defects. 

251 """ 

252 retDefectList = self.__class__() 

253 for defect in self: 

254 bbox = defect.getBBox() 

255 dimensions = bbox.getDimensions() 

256 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()), 

257 lsst.geom.Extent2I(dimensions[1], dimensions[0])) 

258 retDefectList.append(nbbox) 

259 return retDefectList 

260 

261 def maskPixels(self, mask, maskName="BAD"): 

262 """Set mask plane based on these defects. 

263 

264 Parameters 

265 ---------- 

266 maskedImage : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Mask` 

267 Image to process. Only the mask plane is updated. 

268 maskName : str, optional 

269 Mask plane name to use. 

270 """ 

271 # mask bad pixels 

272 if hasattr(mask, "getMask"): 

273 mask = mask.getMask() 

274 bitmask = mask.getPlaneBitMask(maskName) 

275 for defect in self: 

276 bbox = defect.getBBox() 

277 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask) 

278 

279 def updateCounters(self, columns=None, hot=None, cold=None): 

280 """Update metadata with pixel and column counts. 

281 

282 Parameters 

283 ---------- 

284 columns : `int`, optional 

285 Number of full columns masked. 

286 hot : `dict` [`str`, `int`], optional 

287 Dictionary with the count of hot pixels, indexed by amplifier name. 

288 cold : `dict` [`str`, `int`], optional 

289 Dictionary with the count of hot pixels, indexed by amplifier name. 

290 """ 

291 mdSupplemental = dict() 

292 if columns: 

293 mdSupplemental["LSST CALIB DEFECTS N_BAD_COLUMNS"] = columns 

294 if hot: 

295 for amp, count in hot.items(): 

296 mdSupplemental[f"LSST CALIB DEFECTS {amp} N_HOT"] = count 

297 if cold: 

298 for amp, count in cold.items(): 

299 mdSupplemental[f"LSST CALIB DEFECTS {amp} N_COLD"] = count 

300 self.getMetadata().update(mdSupplemental) 

301 

302 def toFitsRegionTable(self): 

303 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the 

304 FITS region standard. 

305 

306 Returns 

307 ------- 

308 table : `lsst.afw.table.BaseCatalog` 

309 Defects in tabular form. 

310 

311 Notes 

312 ----- 

313 The table created uses the 

314 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

315 definition tabular format. The ``X`` and ``Y`` coordinates are 

316 converted to FITS Physical coordinates that have origin pixel (1, 1) 

317 rather than the (0, 0) used in LSST software. 

318 """ 

319 self.updateMetadata() 

320 nrows = len(self._defects) 

321 

322 if nrows: 

323 # Adding entire columns is more efficient than adding 

324 # each element separately 

325 xCol = [] 

326 yCol = [] 

327 rCol = [] 

328 shapes = [] 

329 for i, defect in enumerate(self._defects): 

330 box = defect.getBBox() 

331 center = box.getCenter() 

332 # Correct for the FITS 1-based offset 

333 xCol.append(center.getX() + 1.0) 

334 yCol.append(center.getY() + 1.0) 

335 

336 width = box.width 

337 height = box.height 

338 

339 if width == 1 and height == 1: 

340 # Call this a point 

341 shapeType = "POINT" 

342 else: 

343 shapeType = "BOX" 

344 

345 # Strings have to be added per row 

346 shapes.append(shapeType) 

347 

348 rCol.append(np.array([width, height], dtype=np.float64)) 

349 

350 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes, 

351 'R': rCol, 'ROTANG': np.zeros(nrows), 

352 'COMPONENT': np.arange(nrows)}) 

353 table.meta = self.getMetadata().toDict() 

354 return table 

355 

356 @classmethod 

357 def fromDict(cls, dictionary): 

358 """Construct a calibration from a dictionary of properties. 

359 

360 Must be implemented by the specific calibration subclasses. 

361 

362 Parameters 

363 ---------- 

364 dictionary : `dict` 

365 Dictionary of properties. 

366 

367 Returns 

368 ------- 

369 calib : `lsst.ip.isr.CalibType` 

370 Constructed calibration. 

371 

372 Raises 

373 ------ 

374 RuntimeError 

375 Raised if the supplied dictionary is for a different 

376 calibration. 

377 """ 

378 calib = cls() 

379 

380 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

381 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, " 

382 f"found {dictionary['metadata']['OBSTYPE']}") 

383 

384 calib.setMetadata(dictionary['metadata']) 

385 calib.calibInfoFromDict(dictionary) 

386 

387 xCol = dictionary['x0'] 

388 yCol = dictionary['y0'] 

389 widthCol = dictionary['width'] 

390 heightCol = dictionary['height'] 

391 

392 with calib.bulk_update: 

393 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol): 

394 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0), 

395 lsst.geom.Extent2I(width, height))) 

396 return calib 

397 

398 def toDict(self): 

399 """Return a dictionary containing the calibration properties. 

400 

401 The dictionary should be able to be round-tripped through 

402 `fromDict`. 

403 

404 Returns 

405 ------- 

406 dictionary : `dict` 

407 Dictionary of properties. 

408 """ 

409 self.updateMetadata() 

410 

411 outDict = {} 

412 metadata = self.getMetadata() 

413 outDict['metadata'] = metadata 

414 

415 xCol = [] 

416 yCol = [] 

417 widthCol = [] 

418 heightCol = [] 

419 

420 nrows = len(self._defects) 

421 if nrows: 

422 for defect in self._defects: 

423 box = defect.getBBox() 

424 xCol.append(box.getBeginX()) 

425 yCol.append(box.getBeginY()) 

426 widthCol.append(box.getWidth()) 

427 heightCol.append(box.getHeight()) 

428 

429 outDict['x0'] = xCol 

430 outDict['y0'] = yCol 

431 outDict['width'] = widthCol 

432 outDict['height'] = heightCol 

433 

434 return outDict 

435 

436 def toTable(self): 

437 """Convert defects to a simple table form that we use to write 

438 to text files. 

439 

440 Returns 

441 ------- 

442 table : `lsst.afw.table.BaseCatalog` 

443 Defects in simple tabular form. 

444 

445 Notes 

446 ----- 

447 These defect tables are used as the human readable definitions 

448 of defects in calibration data definition repositories. The format 

449 is to use four columns defined as follows: 

450 

451 x0 : `int` 

452 X coordinate of bottom left corner of box. 

453 y0 : `int` 

454 Y coordinate of bottom left corner of box. 

455 width : `int` 

456 X extent of the box. 

457 height : `int` 

458 Y extent of the box. 

459 """ 

460 tableList = [] 

461 self.updateMetadata() 

462 

463 xCol = [] 

464 yCol = [] 

465 widthCol = [] 

466 heightCol = [] 

467 

468 nrows = len(self._defects) 

469 if nrows: 

470 for defect in self._defects: 

471 box = defect.getBBox() 

472 xCol.append(box.getBeginX()) 

473 yCol.append(box.getBeginY()) 

474 widthCol.append(box.getWidth()) 

475 heightCol.append(box.getHeight()) 

476 

477 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol}) 

478 inMeta = self.getMetadata().toDict() 

479 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

480 catalog.meta = outMeta 

481 tableList.append(catalog) 

482 

483 return tableList 

484 

485 @staticmethod 

486 def _get_values(values, n=1): 

487 """Retrieve N values from the supplied values. 

488 

489 Parameters 

490 ---------- 

491 values : `numbers.Number` or `list` or `np.array` 

492 Input values. 

493 n : `int` 

494 Number of values to retrieve. 

495 

496 Returns 

497 ------- 

498 vals : `list` or `np.array` or `numbers.Number` 

499 Single value from supplied list if ``n`` is 1, or `list` 

500 containing first ``n`` values from supplied values. 

501 

502 Notes 

503 ----- 

504 Some supplied tables have vectors in some columns that can also 

505 be scalars. This method can be used to get the first number as 

506 a scalar or the first N items from a vector as a vector. 

507 """ 

508 if n == 1: 

509 if isinstance(values, numbers.Number): 

510 return values 

511 else: 

512 return values[0] 

513 

514 return values[:n] 

515 

516 @classmethod 

517 def fromTable(cls, tableList, normalize_on_init=True): 

518 """Construct a `Defects` from the contents of a 

519 `~lsst.afw.table.BaseCatalog`. 

520 

521 Parameters 

522 ---------- 

523 table : `lsst.afw.table.BaseCatalog` 

524 Table with one row per defect. 

525 normalize_on_init : `bool`, optional 

526 If `True`, normalization is applied to the defects listed in the 

527 table to remove duplicates, eliminate overlaps, etc. Otherwise 

528 the defects in the returned object exactly match those in the 

529 table. 

530 

531 Returns 

532 ------- 

533 defects : `Defects` 

534 A `Defects` list. 

535 

536 Notes 

537 ----- 

538 Two table formats are recognized. The first is the 

539 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

540 definition tabular format written by `toFitsRegionTable` where the 

541 pixel origin is corrected from FITS 1-based to a 0-based origin. 

542 The second is the legacy defects format using columns ``x0``, ``y0`` 

543 (bottom left hand pixel of box in 0-based coordinates), ``width`` 

544 and ``height``. 

545 

546 The FITS standard regions can only read BOX, POINT, or ROTBOX with 

547 a zero degree rotation. 

548 """ 

549 table = tableList[0] 

550 defectList = [] 

551 

552 schema = table.columns 

553 # Check schema to see which definitions we have 

554 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema: 

555 # This is a FITS region style table 

556 isFitsRegion = True 

557 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema: 

558 # This is a classic LSST-style defect table 

559 isFitsRegion = False 

560 else: 

561 raise ValueError("Unsupported schema for defects extraction") 

562 

563 for record in table: 

564 if isFitsRegion: 

565 # Coordinates can be arrays (some shapes in the standard 

566 # require this) 

567 # Correct for FITS 1-based origin 

568 xcen = cls._get_values(record['X']) - 1.0 

569 ycen = cls._get_values(record['Y']) - 1.0 

570 shape = record['SHAPE'].upper().rstrip() 

571 if shape == "BOX": 

572 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

573 lsst.geom.Extent2I(cls._get_values(record['R'], 

574 n=2))) 

575 elif shape == "POINT": 

576 # Handle the case where we have an externally created 

577 # FITS file. 

578 box = lsst.geom.Point2I(xcen, ycen) 

579 elif shape == "ROTBOX": 

580 # Astropy regions always writes ROTBOX 

581 rotang = cls._get_values(record['ROTANG']) 

582 # We can support 0 or 90 deg 

583 if math.isclose(rotang % 90.0, 0.0): 

584 # Two values required 

585 r = cls._get_values(record['R'], n=2) 

586 if math.isclose(rotang % 180.0, 0.0): 

587 width = r[0] 

588 height = r[1] 

589 else: 

590 width = r[1] 

591 height = r[0] 

592 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

593 lsst.geom.Extent2I(width, height)) 

594 else: 

595 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle") 

596 continue 

597 else: 

598 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape) 

599 continue 

600 

601 else: 

602 # This is a classic LSST-style defect table 

603 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']), 

604 lsst.geom.Extent2I(record['width'], record['height'])) 

605 

606 defectList.append(box) 

607 

608 defects = cls(defectList, normalize_on_init=normalize_on_init) 

609 newMeta = dict(table.meta) 

610 defects.updateMetadata(setCalibInfo=True, **newMeta) 

611 

612 return defects 

613 

614 @classmethod 

615 def readLsstDefectsFile(cls, filename, normalize_on_init=False): 

616 """Read defects information from a legacy LSST format text file. 

617 

618 Parameters 

619 ---------- 

620 filename : `str` 

621 Name of text file containing the defect information. 

622 

623 normalize_on_init : `bool`, optional 

624 If `True`, normalization is applied to the defects listed in the 

625 table to remove duplicates, eliminate overlaps, etc. Otherwise 

626 the defects in the returned object exactly match those in the 

627 table. 

628 

629 Returns 

630 ------- 

631 defects : `Defects` 

632 The defects. 

633 

634 Notes 

635 ----- 

636 These defect text files are used as the human readable definitions 

637 of defects in calibration data definition repositories. The format 

638 is to use four columns defined as follows: 

639 

640 x0 : `int` 

641 X coordinate of bottom left corner of box. 

642 y0 : `int` 

643 Y coordinate of bottom left corner of box. 

644 width : `int` 

645 X extent of the box. 

646 height : `int` 

647 Y extent of the box. 

648 

649 Files of this format were used historically to represent defects 

650 in simple text form. Use `Defects.readText` and `Defects.writeText` 

651 to use the more modern format. 

652 """ 

653 # Use loadtxt so that ValueError is thrown if the file contains a 

654 # non-integer value. genfromtxt converts bad values to -1. 

655 defect_array = np.loadtxt(filename, 

656 dtype=[("x0", "int"), ("y0", "int"), 

657 ("x_extent", "int"), ("y_extent", "int")]) 

658 

659 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]), 

660 lsst.geom.Extent2I(row["x_extent"], row["y_extent"])) 

661 for row in defect_array) 

662 

663 return cls(defects, normalize_on_init=normalize_on_init) 

664 

665 @classmethod 

666 def fromFootprintList(cls, fpList): 

667 """Compute a defect list from a footprint list, optionally growing 

668 the footprints. 

669 

670 Parameters 

671 ---------- 

672 fpList : `list` of `lsst.afw.detection.Footprint` 

673 Footprint list to process. 

674 

675 Returns 

676 ------- 

677 defects : `Defects` 

678 List of defects. 

679 """ 

680 # normalize_on_init is set to False to avoid recursively calling 

681 # fromMask/fromFootprintList in Defects.__init__. 

682 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp) 

683 for fp in fpList), normalize_on_init=False) 

684 

685 @classmethod 

686 def fromMask(cls, mask, maskName): 

687 """Compute a defect list from a specified mask plane. 

688 

689 Parameters 

690 ---------- 

691 mask : `lsst.afw.image.Mask` or `lsst.afw.image.MaskedImage` 

692 Image to process. 

693 maskName : `str` or `list` 

694 Mask plane name, or list of names to convert. 

695 

696 Returns 

697 ------- 

698 defects : `Defects` 

699 Defect list constructed from masked pixels. 

700 """ 

701 if hasattr(mask, "getMask"): 

702 mask = mask.getMask() 

703 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName), 

704 lsst.afw.detection.Threshold.BITMASK) 

705 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints() 

706 return cls.fromFootprintList(fpList)