Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ip_isr. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Support for image defects""" 

22 

23__all__ = ("Defects",) 

24 

25import logging 

26import itertools 

27import contextlib 

28import numpy as np 

29import math 

30import numbers 

31import astropy.table 

32 

33import lsst.geom 

34import lsst.afw.table 

35import lsst.afw.detection 

36import lsst.afw.image 

37import lsst.afw.geom 

38from lsst.meas.algorithms import Defect 

39from .calibType import IsrCalib 

40 

41log = logging.getLogger(__name__) 

42 

43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA" 

44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION" 

45 

46 

47class Defects(IsrCalib): 

48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`. 

49 

50 Parameters 

51 ---------- 

52 defectList : iterable of `lsst.meas.algorithms.Defect` 

53 or `lsst.geom.BoxI`, optional 

54 Collections of defects to apply to the image. 

55 metadata : `lsst.daf.base.PropertyList`, optional 

56 Metadata to associate with the defects. Will be copied and 

57 overwrite existing metadata, if any. If not supplied the existing 

58 metadata will be reset. 

59 normalize_on_init : `bool` 

60 If True, normalization is applied to the defects in ``defectList`` to 

61 remove duplicates, eliminate overlaps, etc. 

62 

63 Notes 

64 ----- 

65 Defects are stored within this collection in a "reduced" or "normalized" 

66 form: rather than simply storing the bounding boxes which are added to the 

67 collection, we eliminate overlaps and duplicates. This normalization 

68 procedure may introduce overhead when adding many new defects; it may be 

69 temporarily disabled using the `Defects.bulk_update` context manager if 

70 necessary. 

71 

72 The attributes stored in this calibration are: 

73 

74 _defects : `list` [`lsst.meas.algorithms.Defect`] 

75 The collection of Defect objects. 

76 """ 

77 

78 """The calibration type used for ingest.""" 

79 _OBSTYPE = "defects" 

80 _SCHEMA = '' 

81 _VERSION = 2.0 

82 

83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs): 

84 self._defects = [] 

85 

86 if defectList is not None: 

87 self._bulk_update = True 

88 for d in defectList: 

89 self.append(d) 

90 self._bulk_update = False 

91 

92 if normalize_on_init: 

93 self._normalize() 

94 

95 super().__init__(**kwargs) 

96 self.requiredAttributes.update(['_defects']) 

97 

98 def _check_value(self, value): 

99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect` 

100 or can be converted to one. 

101 

102 Parameters 

103 ---------- 

104 value : `object` 

105 Value to check. 

106 

107 Returns 

108 ------- 

109 new : `~lsst.meas.algorithms.Defect` 

110 Either the supplied value or a new object derived from it. 

111 

112 Raises 

113 ------ 

114 ValueError 

115 Raised if the supplied value can not be converted to 

116 `~lsst.meas.algorithms.Defect` 

117 """ 

118 if isinstance(value, Defect): 

119 pass 

120 elif isinstance(value, lsst.geom.BoxI): 

121 value = Defect(value) 

122 elif isinstance(value, lsst.geom.PointI): 

123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1))) 

124 elif isinstance(value, lsst.afw.image.DefectBase): 

125 value = Defect(value.getBBox()) 

126 else: 

127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'") 

128 return value 

129 

130 def __len__(self): 

131 return len(self._defects) 

132 

133 def __getitem__(self, index): 

134 return self._defects[index] 

135 

136 def __setitem__(self, index, value): 

137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI` 

138 """ 

139 self._defects[index] = self._check_value(value) 

140 self._normalize() 

141 

142 def __iter__(self): 

143 return iter(self._defects) 

144 

145 def __delitem__(self, index): 

146 del self._defects[index] 

147 

148 def __eq__(self, other): 

149 """Compare if two `Defects` are equal. 

150 

151 Two `Defects` are equal if their bounding boxes are equal and in 

152 the same order. Metadata content is ignored. 

153 """ 

154 super().__eq__(other) 

155 

156 if not isinstance(other, self.__class__): 

157 return False 

158 

159 # checking the bboxes with zip() only works if same length 

160 if len(self) != len(other): 

161 return False 

162 

163 # Assume equal if bounding boxes are equal 

164 for d1, d2 in zip(self, other): 

165 if d1.getBBox() != d2.getBBox(): 

166 return False 

167 

168 return True 

169 

170 def __str__(self): 

171 baseStr = super().__str__(self) 

172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")" 

173 

174 def _normalize(self): 

175 """Recalculate defect bounding boxes for efficiency. 

176 

177 Notes 

178 ----- 

179 Ideally, this would generate the provably-minimal set of bounding 

180 boxes necessary to represent the defects. At present, however, that 

181 doesn't happen: see DM-24781. In the cases of substantial overlaps or 

182 duplication, though, this will produce a much reduced set. 

183 """ 

184 # In bulk-update mode, normalization is a no-op. 

185 if self._bulk_update: 

186 return 

187 

188 # work out the minimum and maximum bounds from all defect regions. 

189 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf') 

190 for defect in self: 

191 bbox = defect.getBBox() 

192 minX = min(minX, bbox.getMinX()) 

193 minY = min(minY, bbox.getMinY()) 

194 maxX = max(maxX, bbox.getMaxX()) 

195 maxY = max(maxY, bbox.getMaxY()) 

196 

197 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY), 

198 lsst.geom.Point2I(maxX, maxY)) 

199 

200 mi = lsst.afw.image.MaskedImageF(region) 

201 self.maskPixels(mi, maskName="BAD") 

202 self._defects = Defects.fromMask(mi, "BAD")._defects 

203 

204 @contextlib.contextmanager 

205 def bulk_update(self): 

206 """Temporarily suspend normalization of the defect list. 

207 """ 

208 self._bulk_update = True 

209 try: 

210 yield 

211 finally: 

212 self._bulk_update = False 

213 self._normalize() 

214 

215 def append(self, value): 

216 self._defects.append(self._check_value(value)) 

217 self._normalize() 

218 

219 def insert(self, index, value): 

220 self._defects.insert(index, self._check_value(value)) 

221 self._normalize() 

222 

223 def copy(self): 

224 """Copy the defects to a new list, creating new defects from the 

225 bounding boxes. 

226 

227 Returns 

228 ------- 

229 new : `Defects` 

230 New list with new `Defect` entries. 

231 

232 Notes 

233 ----- 

234 This is not a shallow copy in that new `Defect` instances are 

235 created from the original bounding boxes. It's also not a deep 

236 copy since the bounding boxes are not recreated. 

237 """ 

238 return self.__class__(d.getBBox() for d in self) 

239 

240 def transpose(self): 

241 """Make a transposed copy of this defect list. 

242 

243 Returns 

244 ------- 

245 retDefectList : `Defects` 

246 Transposed list of defects. 

247 """ 

248 retDefectList = self.__class__() 

249 for defect in self: 

250 bbox = defect.getBBox() 

251 dimensions = bbox.getDimensions() 

252 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()), 

253 lsst.geom.Extent2I(dimensions[1], dimensions[0])) 

254 retDefectList.append(nbbox) 

255 return retDefectList 

256 

257 def maskPixels(self, maskedImage, maskName="BAD"): 

258 """Set mask plane based on these defects. 

259 

260 Parameters 

261 ---------- 

262 maskedImage : `lsst.afw.image.MaskedImage` 

263 Image to process. Only the mask plane is updated. 

264 maskName : str, optional 

265 Mask plane name to use. 

266 """ 

267 # mask bad pixels 

268 mask = maskedImage.getMask() 

269 bitmask = mask.getPlaneBitMask(maskName) 

270 for defect in self: 

271 bbox = defect.getBBox() 

272 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask) 

273 

274 def toFitsRegionTable(self): 

275 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the 

276 FITS region standard. 

277 

278 Returns 

279 ------- 

280 table : `lsst.afw.table.BaseCatalog` 

281 Defects in tabular form. 

282 

283 Notes 

284 ----- 

285 The table created uses the 

286 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

287 definition tabular format. The ``X`` and ``Y`` coordinates are 

288 converted to FITS Physical coordinates that have origin pixel (1, 1) 

289 rather than the (0, 0) used in LSST software. 

290 """ 

291 self.updateMetadata() 

292 nrows = len(self._defects) 

293 

294 if nrows: 

295 # Adding entire columns is more efficient than adding 

296 # each element separately 

297 xCol = [] 

298 yCol = [] 

299 rCol = [] 

300 shapes = [] 

301 for i, defect in enumerate(self._defects): 

302 box = defect.getBBox() 

303 center = box.getCenter() 

304 # Correct for the FITS 1-based offset 

305 xCol.append(center.getX() + 1.0) 

306 yCol.append(center.getY() + 1.0) 

307 

308 width = box.width 

309 height = box.height 

310 

311 if width == 1 and height == 1: 

312 # Call this a point 

313 shapeType = "POINT" 

314 else: 

315 shapeType = "BOX" 

316 

317 # Strings have to be added per row 

318 shapes.append(shapeType) 

319 

320 rCol.append(np.array([width, height], dtype=np.float64)) 

321 

322 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes, 

323 'R': rCol, 'ROTANG': np.zeros(nrows), 

324 'COMPONENT': np.arange(nrows)}) 

325 table.meta = self.getMetadata().toDict() 

326 return table 

327 

328 @classmethod 

329 def fromDict(cls, dictionary): 

330 """Construct a calibration from a dictionary of properties. 

331 

332 Must be implemented by the specific calibration subclasses. 

333 

334 Parameters 

335 ---------- 

336 dictionary : `dict` 

337 Dictionary of properties. 

338 

339 Returns 

340 ------- 

341 calib : `lsst.ip.isr.CalibType` 

342 Constructed calibration. 

343 

344 Raises 

345 ------ 

346 RuntimeError : 

347 Raised if the supplied dictionary is for a different 

348 calibration. 

349 """ 

350 calib = cls() 

351 

352 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']: 

353 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, " 

354 f"found {dictionary['metadata']['OBSTYPE']}") 

355 

356 calib.setMetadata(dictionary['metadata']) 

357 calib.calibInfoFromDict(dictionary) 

358 

359 xCol = dictionary['x0'] 

360 yCol = dictionary['y0'] 

361 widthCol = dictionary['width'] 

362 heightCol = dictionary['height'] 

363 

364 with calib.bulk_update: 

365 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol): 

366 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0), 

367 lsst.geom.Extent2I(width, height))) 

368 return calib 

369 

370 def toDict(self): 

371 """Return a dictionary containing the calibration properties. 

372 

373 The dictionary should be able to be round-tripped through 

374 `fromDict`. 

375 

376 Returns 

377 ------- 

378 dictionary : `dict` 

379 Dictionary of properties. 

380 """ 

381 self.updateMetadata() 

382 

383 outDict = {} 

384 metadata = self.getMetadata() 

385 outDict['metadata'] = metadata 

386 

387 xCol = [] 

388 yCol = [] 

389 widthCol = [] 

390 heightCol = [] 

391 

392 nrows = len(self._defects) 

393 if nrows: 

394 for defect in self._defects: 

395 box = defect.getBBox() 

396 xCol.append(box.getBeginX()) 

397 yCol.append(box.getBeginY()) 

398 widthCol.append(box.getWidth()) 

399 heightCol.append(box.getHeight()) 

400 

401 outDict['x0'] = xCol 

402 outDict['y0'] = yCol 

403 outDict['width'] = widthCol 

404 outDict['height'] = heightCol 

405 

406 return outDict 

407 

408 def toTable(self): 

409 """Convert defects to a simple table form that we use to write 

410 to text files. 

411 

412 Returns 

413 ------- 

414 table : `lsst.afw.table.BaseCatalog` 

415 Defects in simple tabular form. 

416 

417 Notes 

418 ----- 

419 These defect tables are used as the human readable definitions 

420 of defects in calibration data definition repositories. The format 

421 is to use four columns defined as follows: 

422 

423 x0 : `int` 

424 X coordinate of bottom left corner of box. 

425 y0 : `int` 

426 Y coordinate of bottom left corner of box. 

427 width : `int` 

428 X extent of the box. 

429 height : `int` 

430 Y extent of the box. 

431 """ 

432 tableList = [] 

433 self.updateMetadata() 

434 

435 xCol = [] 

436 yCol = [] 

437 widthCol = [] 

438 heightCol = [] 

439 

440 nrows = len(self._defects) 

441 if nrows: 

442 for defect in self._defects: 

443 box = defect.getBBox() 

444 xCol.append(box.getBeginX()) 

445 yCol.append(box.getBeginY()) 

446 widthCol.append(box.getWidth()) 

447 heightCol.append(box.getHeight()) 

448 

449 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol}) 

450 inMeta = self.getMetadata().toDict() 

451 outMeta = {k: v for k, v in inMeta.items() if v is not None} 

452 catalog.meta = outMeta 

453 tableList.append(catalog) 

454 

455 return tableList 

456 

457 @staticmethod 

458 def _get_values(values, n=1): 

459 """Retrieve N values from the supplied values. 

460 

461 Parameters 

462 ---------- 

463 values : `numbers.Number` or `list` or `np.array` 

464 Input values. 

465 n : `int` 

466 Number of values to retrieve. 

467 

468 Returns 

469 ------- 

470 vals : `list` or `np.array` or `numbers.Number` 

471 Single value from supplied list if ``n`` is 1, or `list` 

472 containing first ``n`` values from supplied values. 

473 

474 Notes 

475 ----- 

476 Some supplied tables have vectors in some columns that can also 

477 be scalars. This method can be used to get the first number as 

478 a scalar or the first N items from a vector as a vector. 

479 """ 

480 if n == 1: 

481 if isinstance(values, numbers.Number): 

482 return values 

483 else: 

484 return values[0] 

485 

486 return values[:n] 

487 

488 @classmethod 

489 def fromTable(cls, tableList, normalize_on_init=True): 

490 """Construct a `Defects` from the contents of a 

491 `~lsst.afw.table.BaseCatalog`. 

492 

493 Parameters 

494 ---------- 

495 table : `lsst.afw.table.BaseCatalog` 

496 Table with one row per defect. 

497 normalize_on_init : `bool`, optional 

498 If `True`, normalization is applied to the defects listed in the 

499 table to remove duplicates, eliminate overlaps, etc. Otherwise 

500 the defects in the returned object exactly match those in the 

501 table. 

502 

503 Returns 

504 ------- 

505 defects : `Defects` 

506 A `Defects` list. 

507 

508 Notes 

509 ----- 

510 Two table formats are recognized. The first is the 

511 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

512 definition tabular format written by `toFitsRegionTable` where the 

513 pixel origin is corrected from FITS 1-based to a 0-based origin. 

514 The second is the legacy defects format using columns ``x0``, ``y0`` 

515 (bottom left hand pixel of box in 0-based coordinates), ``width`` 

516 and ``height``. 

517 

518 The FITS standard regions can only read BOX, POINT, or ROTBOX with 

519 a zero degree rotation. 

520 """ 

521 table = tableList[0] 

522 defectList = [] 

523 

524 schema = table.columns 

525 # Check schema to see which definitions we have 

526 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema: 

527 # This is a FITS region style table 

528 isFitsRegion = True 

529 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema: 

530 # This is a classic LSST-style defect table 

531 isFitsRegion = False 

532 else: 

533 raise ValueError("Unsupported schema for defects extraction") 

534 

535 for record in table: 

536 if isFitsRegion: 

537 # Coordinates can be arrays (some shapes in the standard 

538 # require this) 

539 # Correct for FITS 1-based origin 

540 xcen = cls._get_values(record['X']) - 1.0 

541 ycen = cls._get_values(record['Y']) - 1.0 

542 shape = record['SHAPE'].upper().rstrip() 

543 if shape == "BOX": 

544 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

545 lsst.geom.Extent2I(cls._get_values(record['R'], 

546 n=2))) 

547 elif shape == "POINT": 

548 # Handle the case where we have an externally created 

549 # FITS file. 

550 box = lsst.geom.Point2I(xcen, ycen) 

551 elif shape == "ROTBOX": 

552 # Astropy regions always writes ROTBOX 

553 rotang = cls._get_values(record['ROTANG']) 

554 # We can support 0 or 90 deg 

555 if math.isclose(rotang % 90.0, 0.0): 

556 # Two values required 

557 r = cls._get_values(record['R'], n=2) 

558 if math.isclose(rotang % 180.0, 0.0): 

559 width = r[0] 

560 height = r[1] 

561 else: 

562 width = r[1] 

563 height = r[0] 

564 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

565 lsst.geom.Extent2I(width, height)) 

566 else: 

567 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle") 

568 continue 

569 else: 

570 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape) 

571 continue 

572 

573 else: 

574 # This is a classic LSST-style defect table 

575 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']), 

576 lsst.geom.Extent2I(record['width'], record['height'])) 

577 

578 defectList.append(box) 

579 

580 defects = cls(defectList, normalize_on_init=normalize_on_init) 

581 newMeta = dict(table.meta) 

582 defects.updateMetadata(setCalibInfo=True, **newMeta) 

583 

584 return defects 

585 

586 @classmethod 

587 def readLsstDefectsFile(cls, filename, normalize_on_init=False): 

588 """Read defects information from a legacy LSST format text file. 

589 

590 Parameters 

591 ---------- 

592 filename : `str` 

593 Name of text file containing the defect information. 

594 

595 normalize_on_init : `bool`, optional 

596 If `True`, normalization is applied to the defects listed in the 

597 table to remove duplicates, eliminate overlaps, etc. Otherwise 

598 the defects in the returned object exactly match those in the 

599 table. 

600 

601 Returns 

602 ------- 

603 defects : `Defects` 

604 The defects. 

605 

606 Notes 

607 ----- 

608 These defect text files are used as the human readable definitions 

609 of defects in calibration data definition repositories. The format 

610 is to use four columns defined as follows: 

611 

612 x0 : `int` 

613 X coordinate of bottom left corner of box. 

614 y0 : `int` 

615 Y coordinate of bottom left corner of box. 

616 width : `int` 

617 X extent of the box. 

618 height : `int` 

619 Y extent of the box. 

620 

621 Files of this format were used historically to represent defects 

622 in simple text form. Use `Defects.readText` and `Defects.writeText` 

623 to use the more modern format. 

624 """ 

625 # Use loadtxt so that ValueError is thrown if the file contains a 

626 # non-integer value. genfromtxt converts bad values to -1. 

627 defect_array = np.loadtxt(filename, 

628 dtype=[("x0", "int"), ("y0", "int"), 

629 ("x_extent", "int"), ("y_extent", "int")]) 

630 

631 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]), 

632 lsst.geom.Extent2I(row["x_extent"], row["y_extent"])) 

633 for row in defect_array) 

634 

635 return cls(defects, normalize_on_init=normalize_on_init) 

636 

637 @classmethod 

638 def fromFootprintList(cls, fpList): 

639 """Compute a defect list from a footprint list, optionally growing 

640 the footprints. 

641 

642 Parameters 

643 ---------- 

644 fpList : `list` of `lsst.afw.detection.Footprint` 

645 Footprint list to process. 

646 

647 Returns 

648 ------- 

649 defects : `Defects` 

650 List of defects. 

651 """ 

652 # normalize_on_init is set to False to avoid recursively calling 

653 # fromMask/fromFootprintList in Defects.__init__. 

654 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp) 

655 for fp in fpList), normalize_on_init=False) 

656 

657 @classmethod 

658 def fromMask(cls, maskedImage, maskName): 

659 """Compute a defect list from a specified mask plane. 

660 

661 Parameters 

662 ---------- 

663 maskedImage : `lsst.afw.image.MaskedImage` 

664 Image to process. 

665 maskName : `str` or `list` 

666 Mask plane name, or list of names to convert. 

667 

668 Returns 

669 ------- 

670 defects : `Defects` 

671 Defect list constructed from masked pixels. 

672 """ 

673 mask = maskedImage.getMask() 

674 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName), 

675 lsst.afw.detection.Threshold.BITMASK) 

676 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints() 

677 return cls.fromFootprintList(fpList)