Coverage for python/lsst/ip/isr/defects.py: 16%

256 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-01 03:20 -0700

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Support for image defects""" 

22 

23__all__ = ("Defects",) 

24 

25import logging 

26import itertools 

27import contextlib 

28import numpy as np 

29import math 

30import numbers 

31import astropy.table 

32 

33import lsst.geom 

34import lsst.afw.table 

35import lsst.afw.detection 

36import lsst.afw.image 

37import lsst.afw.geom 

38from lsst.meas.algorithms import Defect 

39from .calibType import IsrCalib 

40 

41log = logging.getLogger(__name__) 

42 

43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA" 

44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION" 

45 

46 

47class Defects(IsrCalib): 

48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`. 

49 

50 Parameters 

51 ---------- 

52 defectList : iterable, optional 

53 Collections of defects to apply to the image. Can be an iterable of 

54 `lsst.meas.algorithms.Defect` or `lsst.geom.BoxI`. 

55 metadata : `lsst.daf.base.PropertyList`, optional 

56 Metadata to associate with the defects. Will be copied and 

57 overwrite existing metadata, if any. If not supplied the existing 

58 metadata will be reset. 

59 normalize_on_init : `bool` 

60 If True, normalization is applied to the defects in ``defectList`` to 

61 remove duplicates, eliminate overlaps, etc. 

62 

63 Notes 

64 ----- 

65 Defects are stored within this collection in a "reduced" or "normalized" 

66 form: rather than simply storing the bounding boxes which are added to the 

67 collection, we eliminate overlaps and duplicates. This normalization 

68 procedure may introduce overhead when adding many new defects; it may be 

69 temporarily disabled using the `Defects.bulk_update` context manager if 

70 necessary. 

71 

72 The attributes stored in this calibration are: 

73 

74 _defects : `list` [`lsst.meas.algorithms.Defect`] 

75 The collection of Defect objects. 

76 """ 

77 

78 """The calibration type used for ingest.""" 

79 _OBSTYPE = "defects" 

80 _SCHEMA = '' 

81 _VERSION = 2.0 

82 

83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs): 

84 self._defects = [] 

85 

86 if defectList is not None: 

87 self._bulk_update = True 

88 for d in defectList: 

89 self.append(d) 

90 self._bulk_update = False 

91 

92 if normalize_on_init: 

93 self._normalize() 

94 

95 super().__init__(**kwargs) 

96 self.requiredAttributes.update(['_defects']) 

97 

98 def _check_value(self, value): 

99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect` 

100 or can be converted to one. 

101 

102 Parameters 

103 ---------- 

104 value : `object` 

105 Value to check. 

106 

107 Returns 

108 ------- 

109 new : `~lsst.meas.algorithms.Defect` 

110 Either the supplied value or a new object derived from it. 

111 

112 Raises 

113 ------ 

114 ValueError 

115 Raised if the supplied value can not be converted to 

116 `~lsst.meas.algorithms.Defect` 

117 """ 

118 if isinstance(value, Defect): 

119 pass 

120 elif isinstance(value, lsst.geom.BoxI): 

121 value = Defect(value) 

122 elif isinstance(value, lsst.geom.PointI): 

123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1))) 

124 elif isinstance(value, lsst.afw.image.DefectBase): 

125 value = Defect(value.getBBox()) 

126 else: 

127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'") 

128 return value 

129 

130 def __len__(self): 

131 return len(self._defects) 

132 

133 def __getitem__(self, index): 

134 return self._defects[index] 

135 

136 def __setitem__(self, index, value): 

137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI` 

138 """ 

139 self._defects[index] = self._check_value(value) 

140 self._normalize() 

141 

142 def __iter__(self): 

143 return iter(self._defects) 

144 

145 def __delitem__(self, index): 

146 del self._defects[index] 

147 

148 def __eq__(self, other): 

149 """Compare if two `Defects` are equal. 

150 

151 Two `Defects` are equal if their bounding boxes are equal and in 

152 the same order. Metadata content is ignored. 

153 """ 

154 super().__eq__(other) 

155 

156 if not isinstance(other, self.__class__): 

157 return False 

158 

159 # checking the bboxes with zip() only works if same length 

160 if len(self) != len(other): 

161 return False 

162 

163 # Assume equal if bounding boxes are equal 

164 for d1, d2 in zip(self, other): 

165 if d1.getBBox() != d2.getBBox(): 

166 return False 

167 

168 return True 

169 

170 def __str__(self): 

171 baseStr = super().__str__() 

172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")" 

173 

174 def _normalize(self): 

175 """Recalculate defect bounding boxes for efficiency. 

176 

177 Notes 

178 ----- 

179 Ideally, this would generate the provably-minimal set of bounding 

180 boxes necessary to represent the defects. At present, however, that 

181 doesn't happen: see DM-24781. In the cases of substantial overlaps or 

182 duplication, though, this will produce a much reduced set. 

183 """ 

184 # In bulk-update mode, normalization is a no-op. 

185 if self._bulk_update: 

186 return 

187 

188 # If we have no defects, there is nothing to normalize. 

189 if len(self) == 0: 

190 return 

191 

192 # work out the minimum and maximum bounds from all defect regions. 

193 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf') 

194 for defect in self: 

195 bbox = defect.getBBox() 

196 minX = min(minX, bbox.getMinX()) 

197 minY = min(minY, bbox.getMinY()) 

198 maxX = max(maxX, bbox.getMaxX()) 

199 maxY = max(maxY, bbox.getMaxY()) 

200 

201 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY), 

202 lsst.geom.Point2I(maxX, maxY)) 

203 

204 mi = lsst.afw.image.MaskedImageF(region) 

205 self.maskPixels(mi, maskName="BAD") 

206 self._defects = Defects.fromMask(mi, "BAD")._defects 

207 

208 @contextlib.contextmanager 

209 def bulk_update(self): 

210 """Temporarily suspend normalization of the defect list. 

211 """ 

212 self._bulk_update = True 

213 try: 

214 yield 

215 finally: 

216 self._bulk_update = False 

217 self._normalize() 

218 

219 def append(self, value): 

220 self._defects.append(self._check_value(value)) 

221 self._normalize() 

222 

223 def insert(self, index, value): 

224 self._defects.insert(index, self._check_value(value)) 

225 self._normalize() 

226 

227 def copy(self): 

228 """Copy the defects to a new list, creating new defects from the 

229 bounding boxes. 

230 

231 Returns 

232 ------- 

233 new : `Defects` 

234 New list with new `Defect` entries. 

235 

236 Notes 

237 ----- 

238 This is not a shallow copy in that new `Defect` instances are 

239 created from the original bounding boxes. It's also not a deep 

240 copy since the bounding boxes are not recreated. 

241 """ 

242 return self.__class__(d.getBBox() for d in self) 

243 

244 def transpose(self): 

245 """Make a transposed copy of this defect list. 

246 

247 Returns 

248 ------- 

249 retDefectList : `Defects` 

250 Transposed list of defects. 

251 """ 

252 retDefectList = self.__class__() 

253 for defect in self: 

254 bbox = defect.getBBox() 

255 dimensions = bbox.getDimensions() 

256 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()), 

257 lsst.geom.Extent2I(dimensions[1], dimensions[0])) 

258 retDefectList.append(nbbox) 

259 return retDefectList 

260 

261 def maskPixels(self, maskedImage, maskName="BAD"): 

262 """Set mask plane based on these defects. 

263 

264 Parameters 

265 ---------- 

266 maskedImage : `lsst.afw.image.MaskedImage` 

267 Image to process. Only the mask plane is updated. 

268 maskName : str, optional 

269 Mask plane name to use. 

270 """ 

271 # mask bad pixels 

272 mask = maskedImage.getMask() 

273 bitmask = mask.getPlaneBitMask(maskName) 

274 for defect in self: 

275 bbox = defect.getBBox() 

276 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask) 

277 

278 def toFitsRegionTable(self): 

279 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the 

280 FITS region standard. 

281 

282 Returns 

283 ------- 

284 table : `lsst.afw.table.BaseCatalog` 

285 Defects in tabular form. 

286 

287 Notes 

288 ----- 

289 The table created uses the 

290 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

291 definition tabular format. The ``X`` and ``Y`` coordinates are 

292 converted to FITS Physical coordinates that have origin pixel (1, 1) 

293 rather than the (0, 0) used in LSST software. 

294 """ 

295 self.updateMetadata() 

296 nrows = len(self._defects) 

297 

298 if nrows: 

299 # Adding entire columns is more efficient than adding 

300 # each element separately 

301 xCol = [] 

302 yCol = [] 

303 rCol = [] 

304 shapes = [] 

305 for i, defect in enumerate(self._defects): 

306 box = defect.getBBox() 

307 center = box.getCenter() 

308 # Correct for the FITS 1-based offset 

309 xCol.append(center.getX() + 1.0) 

310 yCol.append(center.getY() + 1.0) 

311 

312 width = box.width 

313 height = box.height 

314 

315 if width == 1 and height == 1: 

316 # Call this a point 

317 shapeType = "POINT" 

318 else: 

319 shapeType = "BOX" 

320 

321 # Strings have to be added per row 

322 shapes.append(shapeType) 

323 

324 rCol.append(np.array([width, height], dtype=np.float64)) 

325 

326 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes, 

327 'R': rCol, 'ROTANG': np.zeros(nrows), 

328 'COMPONENT': np.arange(nrows)}) 

329 table.meta = self.getMetadata().toDict() 

330 return table 

331 

332 @classmethod 

333 def fromDict(cls, dictionary): 

334 """Construct a calibration from a dictionary of properties. 

335 

336 Must be implemented by the specific calibration subclasses. 

337 

338 Parameters 

339 ---------- 

340 dictionary : `dict` 

341 Dictionary of properties. 

342 

343 Returns 

344 ------- 

345 calib : `lsst.ip.isr.CalibType` 

346 Constructed calibration. 

347 

348 Raises 

349 ------ 

350 RuntimeError 

351 Raised if the supplied dictionary is for a different 

352 calibration. 

353 """ 

354 calib = cls() 

355 

356 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

357 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, " 

358 f"found {dictionary['metadata']['OBSTYPE']}") 

359 

360 calib.setMetadata(dictionary['metadata']) 

361 calib.calibInfoFromDict(dictionary) 

362 

363 xCol = dictionary['x0'] 

364 yCol = dictionary['y0'] 

365 widthCol = dictionary['width'] 

366 heightCol = dictionary['height'] 

367 

368 with calib.bulk_update: 

369 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol): 

370 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0), 

371 lsst.geom.Extent2I(width, height))) 

372 return calib 

373 

374 def toDict(self): 

375 """Return a dictionary containing the calibration properties. 

376 

377 The dictionary should be able to be round-tripped through 

378 `fromDict`. 

379 

380 Returns 

381 ------- 

382 dictionary : `dict` 

383 Dictionary of properties. 

384 """ 

385 self.updateMetadata() 

386 

387 outDict = {} 

388 metadata = self.getMetadata() 

389 outDict['metadata'] = metadata 

390 

391 xCol = [] 

392 yCol = [] 

393 widthCol = [] 

394 heightCol = [] 

395 

396 nrows = len(self._defects) 

397 if nrows: 

398 for defect in self._defects: 

399 box = defect.getBBox() 

400 xCol.append(box.getBeginX()) 

401 yCol.append(box.getBeginY()) 

402 widthCol.append(box.getWidth()) 

403 heightCol.append(box.getHeight()) 

404 

405 outDict['x0'] = xCol 

406 outDict['y0'] = yCol 

407 outDict['width'] = widthCol 

408 outDict['height'] = heightCol 

409 

410 return outDict 

411 

412 def toTable(self): 

413 """Convert defects to a simple table form that we use to write 

414 to text files. 

415 

416 Returns 

417 ------- 

418 table : `lsst.afw.table.BaseCatalog` 

419 Defects in simple tabular form. 

420 

421 Notes 

422 ----- 

423 These defect tables are used as the human readable definitions 

424 of defects in calibration data definition repositories. The format 

425 is to use four columns defined as follows: 

426 

427 x0 : `int` 

428 X coordinate of bottom left corner of box. 

429 y0 : `int` 

430 Y coordinate of bottom left corner of box. 

431 width : `int` 

432 X extent of the box. 

433 height : `int` 

434 Y extent of the box. 

435 """ 

436 tableList = [] 

437 self.updateMetadata() 

438 

439 xCol = [] 

440 yCol = [] 

441 widthCol = [] 

442 heightCol = [] 

443 

444 nrows = len(self._defects) 

445 if nrows: 

446 for defect in self._defects: 

447 box = defect.getBBox() 

448 xCol.append(box.getBeginX()) 

449 yCol.append(box.getBeginY()) 

450 widthCol.append(box.getWidth()) 

451 heightCol.append(box.getHeight()) 

452 

453 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol}) 

454 inMeta = self.getMetadata().toDict() 

455 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

456 catalog.meta = outMeta 

457 tableList.append(catalog) 

458 

459 return tableList 

460 

461 @staticmethod 

462 def _get_values(values, n=1): 

463 """Retrieve N values from the supplied values. 

464 

465 Parameters 

466 ---------- 

467 values : `numbers.Number` or `list` or `np.array` 

468 Input values. 

469 n : `int` 

470 Number of values to retrieve. 

471 

472 Returns 

473 ------- 

474 vals : `list` or `np.array` or `numbers.Number` 

475 Single value from supplied list if ``n`` is 1, or `list` 

476 containing first ``n`` values from supplied values. 

477 

478 Notes 

479 ----- 

480 Some supplied tables have vectors in some columns that can also 

481 be scalars. This method can be used to get the first number as 

482 a scalar or the first N items from a vector as a vector. 

483 """ 

484 if n == 1: 

485 if isinstance(values, numbers.Number): 

486 return values 

487 else: 

488 return values[0] 

489 

490 return values[:n] 

491 

492 @classmethod 

493 def fromTable(cls, tableList, normalize_on_init=True): 

494 """Construct a `Defects` from the contents of a 

495 `~lsst.afw.table.BaseCatalog`. 

496 

497 Parameters 

498 ---------- 

499 table : `lsst.afw.table.BaseCatalog` 

500 Table with one row per defect. 

501 normalize_on_init : `bool`, optional 

502 If `True`, normalization is applied to the defects listed in the 

503 table to remove duplicates, eliminate overlaps, etc. Otherwise 

504 the defects in the returned object exactly match those in the 

505 table. 

506 

507 Returns 

508 ------- 

509 defects : `Defects` 

510 A `Defects` list. 

511 

512 Notes 

513 ----- 

514 Two table formats are recognized. The first is the 

515 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

516 definition tabular format written by `toFitsRegionTable` where the 

517 pixel origin is corrected from FITS 1-based to a 0-based origin. 

518 The second is the legacy defects format using columns ``x0``, ``y0`` 

519 (bottom left hand pixel of box in 0-based coordinates), ``width`` 

520 and ``height``. 

521 

522 The FITS standard regions can only read BOX, POINT, or ROTBOX with 

523 a zero degree rotation. 

524 """ 

525 table = tableList[0] 

526 defectList = [] 

527 

528 schema = table.columns 

529 # Check schema to see which definitions we have 

530 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema: 

531 # This is a FITS region style table 

532 isFitsRegion = True 

533 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema: 

534 # This is a classic LSST-style defect table 

535 isFitsRegion = False 

536 else: 

537 raise ValueError("Unsupported schema for defects extraction") 

538 

539 for record in table: 

540 if isFitsRegion: 

541 # Coordinates can be arrays (some shapes in the standard 

542 # require this) 

543 # Correct for FITS 1-based origin 

544 xcen = cls._get_values(record['X']) - 1.0 

545 ycen = cls._get_values(record['Y']) - 1.0 

546 shape = record['SHAPE'].upper().rstrip() 

547 if shape == "BOX": 

548 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

549 lsst.geom.Extent2I(cls._get_values(record['R'], 

550 n=2))) 

551 elif shape == "POINT": 

552 # Handle the case where we have an externally created 

553 # FITS file. 

554 box = lsst.geom.Point2I(xcen, ycen) 

555 elif shape == "ROTBOX": 

556 # Astropy regions always writes ROTBOX 

557 rotang = cls._get_values(record['ROTANG']) 

558 # We can support 0 or 90 deg 

559 if math.isclose(rotang % 90.0, 0.0): 

560 # Two values required 

561 r = cls._get_values(record['R'], n=2) 

562 if math.isclose(rotang % 180.0, 0.0): 

563 width = r[0] 

564 height = r[1] 

565 else: 

566 width = r[1] 

567 height = r[0] 

568 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

569 lsst.geom.Extent2I(width, height)) 

570 else: 

571 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle") 

572 continue 

573 else: 

574 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape) 

575 continue 

576 

577 else: 

578 # This is a classic LSST-style defect table 

579 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']), 

580 lsst.geom.Extent2I(record['width'], record['height'])) 

581 

582 defectList.append(box) 

583 

584 defects = cls(defectList, normalize_on_init=normalize_on_init) 

585 newMeta = dict(table.meta) 

586 defects.updateMetadata(setCalibInfo=True, **newMeta) 

587 

588 return defects 

589 

590 @classmethod 

591 def readLsstDefectsFile(cls, filename, normalize_on_init=False): 

592 """Read defects information from a legacy LSST format text file. 

593 

594 Parameters 

595 ---------- 

596 filename : `str` 

597 Name of text file containing the defect information. 

598 

599 normalize_on_init : `bool`, optional 

600 If `True`, normalization is applied to the defects listed in the 

601 table to remove duplicates, eliminate overlaps, etc. Otherwise 

602 the defects in the returned object exactly match those in the 

603 table. 

604 

605 Returns 

606 ------- 

607 defects : `Defects` 

608 The defects. 

609 

610 Notes 

611 ----- 

612 These defect text files are used as the human readable definitions 

613 of defects in calibration data definition repositories. The format 

614 is to use four columns defined as follows: 

615 

616 x0 : `int` 

617 X coordinate of bottom left corner of box. 

618 y0 : `int` 

619 Y coordinate of bottom left corner of box. 

620 width : `int` 

621 X extent of the box. 

622 height : `int` 

623 Y extent of the box. 

624 

625 Files of this format were used historically to represent defects 

626 in simple text form. Use `Defects.readText` and `Defects.writeText` 

627 to use the more modern format. 

628 """ 

629 # Use loadtxt so that ValueError is thrown if the file contains a 

630 # non-integer value. genfromtxt converts bad values to -1. 

631 defect_array = np.loadtxt(filename, 

632 dtype=[("x0", "int"), ("y0", "int"), 

633 ("x_extent", "int"), ("y_extent", "int")]) 

634 

635 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]), 

636 lsst.geom.Extent2I(row["x_extent"], row["y_extent"])) 

637 for row in defect_array) 

638 

639 return cls(defects, normalize_on_init=normalize_on_init) 

640 

641 @classmethod 

642 def fromFootprintList(cls, fpList): 

643 """Compute a defect list from a footprint list, optionally growing 

644 the footprints. 

645 

646 Parameters 

647 ---------- 

648 fpList : `list` of `lsst.afw.detection.Footprint` 

649 Footprint list to process. 

650 

651 Returns 

652 ------- 

653 defects : `Defects` 

654 List of defects. 

655 """ 

656 # normalize_on_init is set to False to avoid recursively calling 

657 # fromMask/fromFootprintList in Defects.__init__. 

658 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp) 

659 for fp in fpList), normalize_on_init=False) 

660 

661 @classmethod 

662 def fromMask(cls, maskedImage, maskName): 

663 """Compute a defect list from a specified mask plane. 

664 

665 Parameters 

666 ---------- 

667 maskedImage : `lsst.afw.image.MaskedImage` 

668 Image to process. 

669 maskName : `str` or `list` 

670 Mask plane name, or list of names to convert. 

671 

672 Returns 

673 ------- 

674 defects : `Defects` 

675 Defect list constructed from masked pixels. 

676 """ 

677 mask = maskedImage.getMask() 

678 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName), 

679 lsst.afw.detection.Threshold.BITMASK) 

680 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints() 

681 return cls.fromFootprintList(fpList)