Coverage for python/lsst/ip/isr/defects.py: 19%

258 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 09:51 +0000

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Support for image defects""" 

22 

23__all__ = ("Defects",) 

24 

25import logging 

26import itertools 

27import contextlib 

28import numpy as np 

29import math 

30import numbers 

31import astropy.table 

32 

33import lsst.geom 

34import lsst.afw.table 

35import lsst.afw.detection 

36import lsst.afw.image 

37import lsst.afw.geom 

38from lsst.meas.algorithms import Defect 

39from .calibType import IsrCalib 

40 

41log = logging.getLogger(__name__) 

42 

43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA" 

44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION" 

45 

46 

47class Defects(IsrCalib): 

48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`. 

49 

50 Parameters 

51 ---------- 

52 defectList : iterable, optional 

53 Collections of defects to apply to the image. Can be an iterable of 

54 `lsst.meas.algorithms.Defect` or `lsst.geom.BoxI`. 

55 metadata : `lsst.daf.base.PropertyList`, optional 

56 Metadata to associate with the defects. Will be copied and 

57 overwrite existing metadata, if any. If not supplied the existing 

58 metadata will be reset. 

59 normalize_on_init : `bool` 

60 If True, normalization is applied to the defects in ``defectList`` to 

61 remove duplicates, eliminate overlaps, etc. 

62 

63 Notes 

64 ----- 

65 Defects are stored within this collection in a "reduced" or "normalized" 

66 form: rather than simply storing the bounding boxes which are added to the 

67 collection, we eliminate overlaps and duplicates. This normalization 

68 procedure may introduce overhead when adding many new defects; it may be 

69 temporarily disabled using the `Defects.bulk_update` context manager if 

70 necessary. 

71 

72 The attributes stored in this calibration are: 

73 

74 _defects : `list` [`lsst.meas.algorithms.Defect`] 

75 The collection of Defect objects. 

76 """ 

77 

78 """The calibration type used for ingest.""" 

79 _OBSTYPE = "defects" 

80 _SCHEMA = '' 

81 _VERSION = 2.0 

82 

83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs): 

84 self._defects = [] 

85 

86 if defectList is not None: 

87 self._bulk_update = True 

88 for d in defectList: 

89 self.append(d) 

90 self._bulk_update = False 

91 

92 if normalize_on_init: 

93 self._normalize() 

94 

95 super().__init__(**kwargs) 

96 self.requiredAttributes.update(['_defects']) 

97 

98 def _check_value(self, value): 

99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect` 

100 or can be converted to one. 

101 

102 Parameters 

103 ---------- 

104 value : `object` 

105 Value to check. 

106 

107 Returns 

108 ------- 

109 new : `~lsst.meas.algorithms.Defect` 

110 Either the supplied value or a new object derived from it. 

111 

112 Raises 

113 ------ 

114 ValueError 

115 Raised if the supplied value can not be converted to 

116 `~lsst.meas.algorithms.Defect` 

117 """ 

118 if isinstance(value, Defect): 

119 pass 

120 elif isinstance(value, lsst.geom.BoxI): 

121 value = Defect(value) 

122 elif isinstance(value, lsst.geom.PointI): 

123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1))) 

124 elif isinstance(value, lsst.afw.image.DefectBase): 

125 value = Defect(value.getBBox()) 

126 else: 

127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'") 

128 return value 

129 

130 def __len__(self): 

131 return len(self._defects) 

132 

133 def __getitem__(self, index): 

134 return self._defects[index] 

135 

136 def __setitem__(self, index, value): 

137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI` 

138 """ 

139 self._defects[index] = self._check_value(value) 

140 self._normalize() 

141 

142 def __iter__(self): 

143 return iter(self._defects) 

144 

145 def __delitem__(self, index): 

146 del self._defects[index] 

147 

148 def __eq__(self, other): 

149 """Compare if two `Defects` are equal. 

150 

151 Two `Defects` are equal if their bounding boxes are equal and in 

152 the same order. Metadata content is ignored. 

153 """ 

154 super().__eq__(other) 

155 

156 if not isinstance(other, self.__class__): 

157 return False 

158 

159 # checking the bboxes with zip() only works if same length 

160 if len(self) != len(other): 

161 return False 

162 

163 # Assume equal if bounding boxes are equal 

164 for d1, d2 in zip(self, other): 

165 if d1.getBBox() != d2.getBBox(): 

166 return False 

167 

168 return True 

169 

170 def __str__(self): 

171 baseStr = super().__str__() 

172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")" 

173 

174 def _normalize(self): 

175 """Recalculate defect bounding boxes for efficiency. 

176 

177 Notes 

178 ----- 

179 Ideally, this would generate the provably-minimal set of bounding 

180 boxes necessary to represent the defects. At present, however, that 

181 doesn't happen: see DM-24781. In the cases of substantial overlaps or 

182 duplication, though, this will produce a much reduced set. 

183 """ 

184 # In bulk-update mode, normalization is a no-op. 

185 if self._bulk_update: 

186 return 

187 

188 # If we have no defects, there is nothing to normalize. 

189 if len(self) == 0: 

190 return 

191 

192 # work out the minimum and maximum bounds from all defect regions. 

193 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf') 

194 for defect in self: 

195 bbox = defect.getBBox() 

196 minX = min(minX, bbox.getMinX()) 

197 minY = min(minY, bbox.getMinY()) 

198 maxX = max(maxX, bbox.getMaxX()) 

199 maxY = max(maxY, bbox.getMaxY()) 

200 

201 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY), 

202 lsst.geom.Point2I(maxX, maxY)) 

203 

204 mask = lsst.afw.image.Mask(region) 

205 self.maskPixels(mask, maskName="BAD") 

206 self._defects = Defects.fromMask(mask, "BAD")._defects 

207 

208 @contextlib.contextmanager 

209 def bulk_update(self): 

210 """Temporarily suspend normalization of the defect list. 

211 """ 

212 self._bulk_update = True 

213 try: 

214 yield 

215 finally: 

216 self._bulk_update = False 

217 self._normalize() 

218 

219 def append(self, value): 

220 self._defects.append(self._check_value(value)) 

221 self._normalize() 

222 

223 def insert(self, index, value): 

224 self._defects.insert(index, self._check_value(value)) 

225 self._normalize() 

226 

227 def copy(self): 

228 """Copy the defects to a new list, creating new defects from the 

229 bounding boxes. 

230 

231 Returns 

232 ------- 

233 new : `Defects` 

234 New list with new `Defect` entries. 

235 

236 Notes 

237 ----- 

238 This is not a shallow copy in that new `Defect` instances are 

239 created from the original bounding boxes. It's also not a deep 

240 copy since the bounding boxes are not recreated. 

241 """ 

242 return self.__class__(d.getBBox() for d in self) 

243 

244 def transpose(self): 

245 """Make a transposed copy of this defect list. 

246 

247 Returns 

248 ------- 

249 retDefectList : `Defects` 

250 Transposed list of defects. 

251 """ 

252 retDefectList = self.__class__() 

253 for defect in self: 

254 bbox = defect.getBBox() 

255 dimensions = bbox.getDimensions() 

256 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()), 

257 lsst.geom.Extent2I(dimensions[1], dimensions[0])) 

258 retDefectList.append(nbbox) 

259 return retDefectList 

260 

261 def maskPixels(self, mask, maskName="BAD"): 

262 """Set mask plane based on these defects. 

263 

264 Parameters 

265 ---------- 

266 maskedImage : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Mask` 

267 Image to process. Only the mask plane is updated. 

268 maskName : str, optional 

269 Mask plane name to use. 

270 """ 

271 # mask bad pixels 

272 if hasattr(mask, "getMask"): 

273 mask = mask.getMask() 

274 bitmask = mask.getPlaneBitMask(maskName) 

275 for defect in self: 

276 bbox = defect.getBBox() 

277 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask) 

278 

279 def toFitsRegionTable(self): 

280 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the 

281 FITS region standard. 

282 

283 Returns 

284 ------- 

285 table : `lsst.afw.table.BaseCatalog` 

286 Defects in tabular form. 

287 

288 Notes 

289 ----- 

290 The table created uses the 

291 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

292 definition tabular format. The ``X`` and ``Y`` coordinates are 

293 converted to FITS Physical coordinates that have origin pixel (1, 1) 

294 rather than the (0, 0) used in LSST software. 

295 """ 

296 self.updateMetadata() 

297 nrows = len(self._defects) 

298 

299 if nrows: 

300 # Adding entire columns is more efficient than adding 

301 # each element separately 

302 xCol = [] 

303 yCol = [] 

304 rCol = [] 

305 shapes = [] 

306 for i, defect in enumerate(self._defects): 

307 box = defect.getBBox() 

308 center = box.getCenter() 

309 # Correct for the FITS 1-based offset 

310 xCol.append(center.getX() + 1.0) 

311 yCol.append(center.getY() + 1.0) 

312 

313 width = box.width 

314 height = box.height 

315 

316 if width == 1 and height == 1: 

317 # Call this a point 

318 shapeType = "POINT" 

319 else: 

320 shapeType = "BOX" 

321 

322 # Strings have to be added per row 

323 shapes.append(shapeType) 

324 

325 rCol.append(np.array([width, height], dtype=np.float64)) 

326 

327 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes, 

328 'R': rCol, 'ROTANG': np.zeros(nrows), 

329 'COMPONENT': np.arange(nrows)}) 

330 table.meta = self.getMetadata().toDict() 

331 return table 

332 

333 @classmethod 

334 def fromDict(cls, dictionary): 

335 """Construct a calibration from a dictionary of properties. 

336 

337 Must be implemented by the specific calibration subclasses. 

338 

339 Parameters 

340 ---------- 

341 dictionary : `dict` 

342 Dictionary of properties. 

343 

344 Returns 

345 ------- 

346 calib : `lsst.ip.isr.CalibType` 

347 Constructed calibration. 

348 

349 Raises 

350 ------ 

351 RuntimeError 

352 Raised if the supplied dictionary is for a different 

353 calibration. 

354 """ 

355 calib = cls() 

356 

357 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

358 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, " 

359 f"found {dictionary['metadata']['OBSTYPE']}") 

360 

361 calib.setMetadata(dictionary['metadata']) 

362 calib.calibInfoFromDict(dictionary) 

363 

364 xCol = dictionary['x0'] 

365 yCol = dictionary['y0'] 

366 widthCol = dictionary['width'] 

367 heightCol = dictionary['height'] 

368 

369 with calib.bulk_update: 

370 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol): 

371 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0), 

372 lsst.geom.Extent2I(width, height))) 

373 return calib 

374 

375 def toDict(self): 

376 """Return a dictionary containing the calibration properties. 

377 

378 The dictionary should be able to be round-tripped through 

379 `fromDict`. 

380 

381 Returns 

382 ------- 

383 dictionary : `dict` 

384 Dictionary of properties. 

385 """ 

386 self.updateMetadata() 

387 

388 outDict = {} 

389 metadata = self.getMetadata() 

390 outDict['metadata'] = metadata 

391 

392 xCol = [] 

393 yCol = [] 

394 widthCol = [] 

395 heightCol = [] 

396 

397 nrows = len(self._defects) 

398 if nrows: 

399 for defect in self._defects: 

400 box = defect.getBBox() 

401 xCol.append(box.getBeginX()) 

402 yCol.append(box.getBeginY()) 

403 widthCol.append(box.getWidth()) 

404 heightCol.append(box.getHeight()) 

405 

406 outDict['x0'] = xCol 

407 outDict['y0'] = yCol 

408 outDict['width'] = widthCol 

409 outDict['height'] = heightCol 

410 

411 return outDict 

412 

413 def toTable(self): 

414 """Convert defects to a simple table form that we use to write 

415 to text files. 

416 

417 Returns 

418 ------- 

419 table : `lsst.afw.table.BaseCatalog` 

420 Defects in simple tabular form. 

421 

422 Notes 

423 ----- 

424 These defect tables are used as the human readable definitions 

425 of defects in calibration data definition repositories. The format 

426 is to use four columns defined as follows: 

427 

428 x0 : `int` 

429 X coordinate of bottom left corner of box. 

430 y0 : `int` 

431 Y coordinate of bottom left corner of box. 

432 width : `int` 

433 X extent of the box. 

434 height : `int` 

435 Y extent of the box. 

436 """ 

437 tableList = [] 

438 self.updateMetadata() 

439 

440 xCol = [] 

441 yCol = [] 

442 widthCol = [] 

443 heightCol = [] 

444 

445 nrows = len(self._defects) 

446 if nrows: 

447 for defect in self._defects: 

448 box = defect.getBBox() 

449 xCol.append(box.getBeginX()) 

450 yCol.append(box.getBeginY()) 

451 widthCol.append(box.getWidth()) 

452 heightCol.append(box.getHeight()) 

453 

454 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol}) 

455 inMeta = self.getMetadata().toDict() 

456 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

457 catalog.meta = outMeta 

458 tableList.append(catalog) 

459 

460 return tableList 

461 

462 @staticmethod 

463 def _get_values(values, n=1): 

464 """Retrieve N values from the supplied values. 

465 

466 Parameters 

467 ---------- 

468 values : `numbers.Number` or `list` or `np.array` 

469 Input values. 

470 n : `int` 

471 Number of values to retrieve. 

472 

473 Returns 

474 ------- 

475 vals : `list` or `np.array` or `numbers.Number` 

476 Single value from supplied list if ``n`` is 1, or `list` 

477 containing first ``n`` values from supplied values. 

478 

479 Notes 

480 ----- 

481 Some supplied tables have vectors in some columns that can also 

482 be scalars. This method can be used to get the first number as 

483 a scalar or the first N items from a vector as a vector. 

484 """ 

485 if n == 1: 

486 if isinstance(values, numbers.Number): 

487 return values 

488 else: 

489 return values[0] 

490 

491 return values[:n] 

492 

493 @classmethod 

494 def fromTable(cls, tableList, normalize_on_init=True): 

495 """Construct a `Defects` from the contents of a 

496 `~lsst.afw.table.BaseCatalog`. 

497 

498 Parameters 

499 ---------- 

500 table : `lsst.afw.table.BaseCatalog` 

501 Table with one row per defect. 

502 normalize_on_init : `bool`, optional 

503 If `True`, normalization is applied to the defects listed in the 

504 table to remove duplicates, eliminate overlaps, etc. Otherwise 

505 the defects in the returned object exactly match those in the 

506 table. 

507 

508 Returns 

509 ------- 

510 defects : `Defects` 

511 A `Defects` list. 

512 

513 Notes 

514 ----- 

515 Two table formats are recognized. The first is the 

516 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

517 definition tabular format written by `toFitsRegionTable` where the 

518 pixel origin is corrected from FITS 1-based to a 0-based origin. 

519 The second is the legacy defects format using columns ``x0``, ``y0`` 

520 (bottom left hand pixel of box in 0-based coordinates), ``width`` 

521 and ``height``. 

522 

523 The FITS standard regions can only read BOX, POINT, or ROTBOX with 

524 a zero degree rotation. 

525 """ 

526 table = tableList[0] 

527 defectList = [] 

528 

529 schema = table.columns 

530 # Check schema to see which definitions we have 

531 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema: 

532 # This is a FITS region style table 

533 isFitsRegion = True 

534 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema: 

535 # This is a classic LSST-style defect table 

536 isFitsRegion = False 

537 else: 

538 raise ValueError("Unsupported schema for defects extraction") 

539 

540 for record in table: 

541 if isFitsRegion: 

542 # Coordinates can be arrays (some shapes in the standard 

543 # require this) 

544 # Correct for FITS 1-based origin 

545 xcen = cls._get_values(record['X']) - 1.0 

546 ycen = cls._get_values(record['Y']) - 1.0 

547 shape = record['SHAPE'].upper().rstrip() 

548 if shape == "BOX": 

549 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

550 lsst.geom.Extent2I(cls._get_values(record['R'], 

551 n=2))) 

552 elif shape == "POINT": 

553 # Handle the case where we have an externally created 

554 # FITS file. 

555 box = lsst.geom.Point2I(xcen, ycen) 

556 elif shape == "ROTBOX": 

557 # Astropy regions always writes ROTBOX 

558 rotang = cls._get_values(record['ROTANG']) 

559 # We can support 0 or 90 deg 

560 if math.isclose(rotang % 90.0, 0.0): 

561 # Two values required 

562 r = cls._get_values(record['R'], n=2) 

563 if math.isclose(rotang % 180.0, 0.0): 

564 width = r[0] 

565 height = r[1] 

566 else: 

567 width = r[1] 

568 height = r[0] 

569 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

570 lsst.geom.Extent2I(width, height)) 

571 else: 

572 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle") 

573 continue 

574 else: 

575 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape) 

576 continue 

577 

578 else: 

579 # This is a classic LSST-style defect table 

580 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']), 

581 lsst.geom.Extent2I(record['width'], record['height'])) 

582 

583 defectList.append(box) 

584 

585 defects = cls(defectList, normalize_on_init=normalize_on_init) 

586 newMeta = dict(table.meta) 

587 defects.updateMetadata(setCalibInfo=True, **newMeta) 

588 

589 return defects 

590 

591 @classmethod 

592 def readLsstDefectsFile(cls, filename, normalize_on_init=False): 

593 """Read defects information from a legacy LSST format text file. 

594 

595 Parameters 

596 ---------- 

597 filename : `str` 

598 Name of text file containing the defect information. 

599 

600 normalize_on_init : `bool`, optional 

601 If `True`, normalization is applied to the defects listed in the 

602 table to remove duplicates, eliminate overlaps, etc. Otherwise 

603 the defects in the returned object exactly match those in the 

604 table. 

605 

606 Returns 

607 ------- 

608 defects : `Defects` 

609 The defects. 

610 

611 Notes 

612 ----- 

613 These defect text files are used as the human readable definitions 

614 of defects in calibration data definition repositories. The format 

615 is to use four columns defined as follows: 

616 

617 x0 : `int` 

618 X coordinate of bottom left corner of box. 

619 y0 : `int` 

620 Y coordinate of bottom left corner of box. 

621 width : `int` 

622 X extent of the box. 

623 height : `int` 

624 Y extent of the box. 

625 

626 Files of this format were used historically to represent defects 

627 in simple text form. Use `Defects.readText` and `Defects.writeText` 

628 to use the more modern format. 

629 """ 

630 # Use loadtxt so that ValueError is thrown if the file contains a 

631 # non-integer value. genfromtxt converts bad values to -1. 

632 defect_array = np.loadtxt(filename, 

633 dtype=[("x0", "int"), ("y0", "int"), 

634 ("x_extent", "int"), ("y_extent", "int")]) 

635 

636 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]), 

637 lsst.geom.Extent2I(row["x_extent"], row["y_extent"])) 

638 for row in defect_array) 

639 

640 return cls(defects, normalize_on_init=normalize_on_init) 

641 

642 @classmethod 

643 def fromFootprintList(cls, fpList): 

644 """Compute a defect list from a footprint list, optionally growing 

645 the footprints. 

646 

647 Parameters 

648 ---------- 

649 fpList : `list` of `lsst.afw.detection.Footprint` 

650 Footprint list to process. 

651 

652 Returns 

653 ------- 

654 defects : `Defects` 

655 List of defects. 

656 """ 

657 # normalize_on_init is set to False to avoid recursively calling 

658 # fromMask/fromFootprintList in Defects.__init__. 

659 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp) 

660 for fp in fpList), normalize_on_init=False) 

661 

662 @classmethod 

663 def fromMask(cls, mask, maskName): 

664 """Compute a defect list from a specified mask plane. 

665 

666 Parameters 

667 ---------- 

668 mask : `lsst.afw.image.Mask` or `lsst.afw.image.MaskedImage` 

669 Image to process. 

670 maskName : `str` or `list` 

671 Mask plane name, or list of names to convert. 

672 

673 Returns 

674 ------- 

675 defects : `Defects` 

676 Defect list constructed from masked pixels. 

677 """ 

678 if hasattr(mask, "getMask"): 

679 mask = mask.getMask() 

680 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName), 

681 lsst.afw.detection.Threshold.BITMASK) 

682 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints() 

683 return cls.fromFootprintList(fpList)