Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23"""Support for image defects""" 

24 

25__all__ = ("Defects",) 

26 

27import logging 

28import itertools 

29import collections.abc 

30import numpy as np 

31import copy 

32import datetime 

33import math 

34import numbers 

35import os.path 

36import astropy.table 

37 

38import lsst.geom 

39import lsst.afw.table 

40import lsst.afw.detection 

41import lsst.afw.image 

42import lsst.afw.geom 

43from lsst.daf.base import PropertyList 

44 

45from . import Defect 

46 

47log = logging.getLogger(__name__) 

48 

49SCHEMA_NAME_KEY = "DEFECTS_SCHEMA" 

50SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION" 

51 

52 

53class Defects(collections.abc.MutableSequence): 

54 """Collection of `lsst.meas.algorithms.Defect`. 

55 

56 Parameters 

57 ---------- 

58 defectList : iterable of `lsst.meas.algorithms.Defect` 

59 or `lsst.geom.BoxI`, optional 

60 Collections of defects to apply to the image. 

61 """ 

62 

63 _OBSTYPE = "defects" 

64 """The calibration type used for ingest.""" 

65 

66 def __init__(self, defectList=None, metadata=None): 

67 self._defects = [] 

68 

69 if metadata is not None: 

70 self._metadata = metadata 

71 else: 

72 self.setMetadata() 

73 

74 if defectList is None: 

75 return 

76 

77 # Ensure that type checking 

78 for d in defectList: 

79 self.append(d) 

80 

81 def _check_value(self, value): 

82 """Check that the supplied value is a `~lsst.meas.algorithms.Defect` 

83 or can be converted to one. 

84 

85 Parameters 

86 ---------- 

87 value : `object` 

88 Value to check. 

89 

90 Returns 

91 ------- 

92 new : `~lsst.meas.algorithms.Defect` 

93 Either the supplied value or a new object derived from it. 

94 

95 Raises 

96 ------ 

97 ValueError 

98 Raised if the supplied value can not be converted to 

99 `~lsst.meas.algorithms.Defect` 

100 """ 

101 if isinstance(value, Defect): 

102 pass 

103 elif isinstance(value, lsst.geom.BoxI): 

104 value = Defect(value) 

105 elif isinstance(value, lsst.geom.PointI): 

106 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1))) 

107 elif isinstance(value, lsst.afw.image.DefectBase): 

108 value = Defect(value.getBBox()) 

109 else: 

110 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'") 

111 return value 

112 

113 def __len__(self): 

114 return len(self._defects) 

115 

116 def __getitem__(self, index): 

117 return self._defects[index] 

118 

119 def __setitem__(self, index, value): 

120 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI` 

121 """ 

122 self._defects[index] = self._check_value(value) 

123 

124 def __iter__(self): 

125 return iter(self._defects) 

126 

127 def __delitem__(self, index): 

128 del self._defects[index] 

129 

130 def __eq__(self, other): 

131 """Compare if two `Defects` are equal. 

132 

133 Two `Defects` are equal if their bounding boxes are equal and in 

134 the same order. Metadata content is ignored. 

135 """ 

136 if not isinstance(other, self.__class__): 

137 return False 

138 

139 # checking the bboxes with zip() only works if same length 

140 if len(self) != len(other): 

141 return False 

142 

143 # Assume equal if bounding boxes are equal 

144 for d1, d2 in zip(self, other): 

145 if d1.getBBox() != d2.getBBox(): 

146 return False 

147 

148 return True 

149 

150 def __str__(self): 

151 return "Defects(" + ",".join(str(d.getBBox()) for d in self) + ")" 

152 

153 def insert(self, index, value): 

154 self._defects.insert(index, self._check_value(value)) 

155 

156 def getMetadata(self): 

157 """Retrieve metadata associated with these `Defects`. 

158 

159 Returns 

160 ------- 

161 meta : `lsst.daf.base.PropertyList` 

162 Metadata. The returned `~lsst.daf.base.PropertyList` can be 

163 modified by the caller and the changes will be written to 

164 external files. 

165 """ 

166 return self._metadata 

167 

168 def setMetadata(self, metadata=None): 

169 """Store a copy of the supplied metadata with the defects. 

170 

171 Parameters 

172 ---------- 

173 metadata : `lsst.daf.base.PropertyList`, optional 

174 Metadata to associate with the defects. Will be copied and 

175 overwrite existing metadata. If not supplied the existing 

176 metadata will be reset. 

177 """ 

178 if metadata is None: 

179 self._metadata = PropertyList() 

180 else: 

181 self._metadata = copy.copy(metadata) 

182 

183 # Ensure that we have the obs type required by calibration ingest 

184 self._metadata["OBSTYPE"] = self._OBSTYPE 

185 

186 def copy(self): 

187 """Copy the defects to a new list, creating new defects from the 

188 bounding boxes. 

189 

190 Returns 

191 ------- 

192 new : `Defects` 

193 New list with new `Defect` entries. 

194 

195 Notes 

196 ----- 

197 This is not a shallow copy in that new `Defect` instances are 

198 created from the original bounding boxes. It's also not a deep 

199 copy since the bounding boxes are not recreated. 

200 """ 

201 return self.__class__(d.getBBox() for d in self) 

202 

203 def transpose(self): 

204 """Make a transposed copy of this defect list. 

205 

206 Returns 

207 ------- 

208 retDefectList : `Defects` 

209 Transposed list of defects. 

210 """ 

211 retDefectList = self.__class__() 

212 for defect in self: 

213 bbox = defect.getBBox() 

214 dimensions = bbox.getDimensions() 

215 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()), 

216 lsst.geom.Extent2I(dimensions[1], dimensions[0])) 

217 retDefectList.append(nbbox) 

218 return retDefectList 

219 

220 def maskPixels(self, maskedImage, maskName="BAD"): 

221 """Set mask plane based on these defects. 

222 

223 Parameters 

224 ---------- 

225 maskedImage : `lsst.afw.image.MaskedImage` 

226 Image to process. Only the mask plane is updated. 

227 maskName : str, optional 

228 Mask plane name to use. 

229 """ 

230 # mask bad pixels 

231 mask = maskedImage.getMask() 

232 bitmask = mask.getPlaneBitMask(maskName) 

233 for defect in self: 

234 bbox = defect.getBBox() 

235 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask) 

236 

237 def toFitsRegionTable(self): 

238 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the 

239 FITS region standard. 

240 

241 Returns 

242 ------- 

243 table : `lsst.afw.table.BaseCatalog` 

244 Defects in tabular form. 

245 

246 Notes 

247 ----- 

248 The table created uses the 

249 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

250 definition tabular format. The ``X`` and ``Y`` coordinates are 

251 converted to FITS Physical coordinates that have origin pixel (1, 1) 

252 rather than the (0, 0) used in LSST software. 

253 """ 

254 

255 nrows = len(self._defects) 

256 

257 schema = lsst.afw.table.Schema() 

258 x = schema.addField("X", type="D", units="pix", doc="X coordinate of center of shape") 

259 y = schema.addField("Y", type="D", units="pix", doc="Y coordinate of center of shape") 

260 shape = schema.addField("SHAPE", type="String", size=16, doc="Shape defined by these values") 

261 r = schema.addField("R", type="ArrayD", size=2, units="pix", doc="Extents") 

262 rotang = schema.addField("ROTANG", type="D", units="deg", doc="Rotation angle") 

263 component = schema.addField("COMPONENT", type="I", doc="Index of this region") 

264 table = lsst.afw.table.BaseCatalog(schema) 

265 table.resize(nrows) 

266 

267 if nrows: 

268 # Adding entire columns is more efficient than adding 

269 # each element separately 

270 xCol = [] 

271 yCol = [] 

272 rCol = [] 

273 

274 for i, defect in enumerate(self._defects): 

275 box = defect.getBBox() 

276 center = box.getCenter() 

277 # Correct for the FITS 1-based offset 

278 xCol.append(center.getX() + 1.0) 

279 yCol.append(center.getY() + 1.0) 

280 

281 width = box.width 

282 height = box.height 

283 

284 if width == 1 and height == 1: 

285 # Call this a point 

286 shapeType = "POINT" 

287 else: 

288 shapeType = "BOX" 

289 

290 # Strings have to be added per row 

291 table[i][shape] = shapeType 

292 

293 rCol.append(np.array([width, height], dtype=np.float64)) 

294 

295 # Assign the columns 

296 table[x] = np.array(xCol, dtype=np.float64) 

297 table[y] = np.array(yCol, dtype=np.float64) 

298 

299 table[r] = np.array(rCol) 

300 table[rotang] = np.zeros(nrows, dtype=np.float64) 

301 table[component] = np.arange(nrows) 

302 

303 # Set some metadata in the table (force OBSTYPE to exist) 

304 metadata = copy.copy(self.getMetadata()) 

305 metadata["OBSTYPE"] = self._OBSTYPE 

306 metadata[SCHEMA_NAME_KEY] = "FITS Region" 

307 metadata[SCHEMA_VERSION_KEY] = 1 

308 table.setMetadata(metadata) 

309 

310 return table 

311 

312 def writeFits(self, *args): 

313 """Write defect list to FITS. 

314 

315 Parameters 

316 ---------- 

317 *args 

318 Arguments to be forwarded to 

319 `lsst.afw.table.BaseCatalog.writeFits`. 

320 """ 

321 table = self.toFitsRegionTable() 

322 

323 # Add some additional headers useful for tracking purposes 

324 metadata = table.getMetadata() 

325 now = datetime.datetime.utcnow() 

326 metadata["DATE"] = now.isoformat() 

327 metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d") 

328 metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip() 

329 

330 table.writeFits(*args) 

331 

332 def toSimpleTable(self): 

333 """Convert defects to a simple table form that we use to write 

334 to text files. 

335 

336 Returns 

337 ------- 

338 table : `lsst.afw.table.BaseCatalog` 

339 Defects in simple tabular form. 

340 

341 Notes 

342 ----- 

343 These defect tables are used as the human readable definitions 

344 of defects in calibration data definition repositories. The format 

345 is to use four columns defined as follows: 

346 

347 x0 : `int` 

348 X coordinate of bottom left corner of box. 

349 y0 : `int` 

350 Y coordinate of bottom left corner of box. 

351 width : `int` 

352 X extent of the box. 

353 height : `int` 

354 Y extent of the box. 

355 """ 

356 schema = lsst.afw.table.Schema() 

357 x = schema.addField("x0", type="I", units="pix", 

358 doc="X coordinate of bottom left corner of box") 

359 y = schema.addField("y0", type="I", units="pix", 

360 doc="Y coordinate of bottom left corner of box") 

361 width = schema.addField("width", type="I", units="pix", 

362 doc="X extent of box") 

363 height = schema.addField("height", type="I", units="pix", 

364 doc="Y extent of box") 

365 table = lsst.afw.table.BaseCatalog(schema) 

366 

367 nrows = len(self._defects) 

368 table.resize(nrows) 

369 

370 if nrows: 

371 

372 xCol = [] 

373 yCol = [] 

374 widthCol = [] 

375 heightCol = [] 

376 

377 for defect in self._defects: 

378 box = defect.getBBox() 

379 xCol.append(box.getBeginX()) 

380 yCol.append(box.getBeginY()) 

381 widthCol.append(box.getWidth()) 

382 heightCol.append(box.getHeight()) 

383 

384 table[x] = np.array(xCol, dtype=np.int64) 

385 table[y] = np.array(yCol, dtype=np.int64) 

386 table[width] = np.array(widthCol, dtype=np.int64) 

387 table[height] = np.array(heightCol, dtype=np.int64) 

388 

389 # Set some metadata in the table (force OBSTYPE to exist) 

390 metadata = copy.copy(self.getMetadata()) 

391 metadata["OBSTYPE"] = self._OBSTYPE 

392 metadata[SCHEMA_NAME_KEY] = "Simple" 

393 metadata[SCHEMA_VERSION_KEY] = 1 

394 table.setMetadata(metadata) 

395 

396 return table 

397 

398 def writeText(self, filename): 

399 """Write the defects out to a text file with the specified name. 

400 

401 Parameters 

402 ---------- 

403 filename : `str` 

404 Name of the file to write. The file extension ".ecsv" will 

405 always be used. 

406 

407 Returns 

408 ------- 

409 used : `str` 

410 The name of the file used to write the data (which may be 

411 different from the supplied name given the change to file 

412 extension). 

413 

414 Notes 

415 ----- 

416 The file is written to ECSV format and will include any metadata 

417 associated with the `Defects`. 

418 """ 

419 

420 # Using astropy table is the easiest way to serialize to ecsv 

421 afwTable = self.toSimpleTable() 

422 table = afwTable.asAstropy() 

423 

424 metadata = afwTable.getMetadata() 

425 now = datetime.datetime.utcnow() 

426 metadata["DATE"] = now.isoformat() 

427 metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d") 

428 metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip() 

429 

430 table.meta = metadata.toDict() 

431 

432 # Force file extension to .ecsv 

433 path, ext = os.path.splitext(filename) 

434 filename = path + ".ecsv" 

435 table.write(filename, format="ascii.ecsv") 

436 return filename 

437 

438 @staticmethod 

439 def _get_values(values, n=1): 

440 """Retrieve N values from the supplied values. 

441 

442 Parameters 

443 ---------- 

444 values : `numbers.Number` or `list` or `np.array` 

445 Input values. 

446 n : `int` 

447 Number of values to retrieve. 

448 

449 Returns 

450 ------- 

451 vals : `list` or `np.array` or `numbers.Number` 

452 Single value from supplied list if ``n`` is 1, or `list` 

453 containing first ``n`` values from supplied values. 

454 

455 Notes 

456 ----- 

457 Some supplied tables have vectors in some columns that can also 

458 be scalars. This method can be used to get the first number as 

459 a scalar or the first N items from a vector as a vector. 

460 """ 

461 if n == 1: 

462 if isinstance(values, numbers.Number): 

463 return values 

464 else: 

465 return values[0] 

466 

467 return values[:n] 

468 

469 @classmethod 

470 def fromTable(cls, table): 

471 """Construct a `Defects` from the contents of a 

472 `~lsst.afw.table.BaseCatalog`. 

473 

474 Parameters 

475 ---------- 

476 table : `lsst.afw.table.BaseCatalog` 

477 Table with one row per defect. 

478 

479 Returns 

480 ------- 

481 defects : `Defects` 

482 A `Defects` list. 

483 

484 Notes 

485 ----- 

486 Two table formats are recognized. The first is the 

487 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_ 

488 definition tabular format written by `toFitsRegionTable` where the 

489 pixel origin is corrected from FITS 1-based to a 0-based origin. 

490 The second is the legacy defects format using columns ``x0``, ``y0`` 

491 (bottom left hand pixel of box in 0-based coordinates), ``width`` 

492 and ``height``. 

493 

494 The FITS standard regions can only read BOX, POINT, or ROTBOX with 

495 a zero degree rotation. 

496 """ 

497 

498 defectList = [] 

499 

500 schema = table.getSchema() 

501 

502 # Check schema to see which definitions we have 

503 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema: 

504 # This is a FITS region style table 

505 isFitsRegion = True 

506 

507 # Preselect the keys 

508 xKey = schema["X"].asKey() 

509 yKey = schema["Y"].asKey() 

510 shapeKey = schema["SHAPE"].asKey() 

511 rKey = schema["R"].asKey() 

512 rotangKey = schema["ROTANG"].asKey() 

513 

514 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema: 

515 # This is a classic LSST-style defect table 

516 isFitsRegion = False 

517 

518 # Preselect the keys 

519 xKey = schema["x0"].asKey() 

520 yKey = schema["y0"].asKey() 

521 widthKey = schema["width"].asKey() 

522 heightKey = schema["height"].asKey() 

523 

524 else: 

525 raise ValueError("Unsupported schema for defects extraction") 

526 

527 for record in table: 

528 

529 if isFitsRegion: 

530 # Coordinates can be arrays (some shapes in the standard 

531 # require this) 

532 # Correct for FITS 1-based origin 

533 xcen = cls._get_values(record[xKey]) - 1.0 

534 ycen = cls._get_values(record[yKey]) - 1.0 

535 shape = record[shapeKey].upper() 

536 if shape == "BOX": 

537 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

538 lsst.geom.Extent2I(cls._get_values(record[rKey], 

539 n=2))) 

540 elif shape == "POINT": 

541 # Handle the case where we have an externally created 

542 # FITS file. 

543 box = lsst.geom.Point2I(xcen, ycen) 

544 elif shape == "ROTBOX": 

545 # Astropy regions always writes ROTBOX 

546 rotang = cls._get_values(record[rotangKey]) 

547 # We can support 0 or 90 deg 

548 if math.isclose(rotang % 90.0, 0.0): 

549 # Two values required 

550 r = cls._get_values(record[rKey], n=2) 

551 if math.isclose(rotang % 180.0, 0.0): 

552 width = r[0] 

553 height = r[1] 

554 else: 

555 width = r[1] 

556 height = r[0] 

557 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen), 

558 lsst.geom.Extent2I(width, height)) 

559 else: 

560 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle") 

561 continue 

562 else: 

563 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape) 

564 continue 

565 

566 else: 

567 # This is a classic LSST-style defect table 

568 box = lsst.geom.Box2I(lsst.geom.Point2I(record[xKey], record[yKey]), 

569 lsst.geom.Extent2I(record[widthKey], record[heightKey])) 

570 

571 defectList.append(box) 

572 

573 defects = cls(defectList) 

574 defects.setMetadata(table.getMetadata()) 

575 

576 # Once read, the schema headers are irrelevant 

577 metadata = defects.getMetadata() 

578 for k in (SCHEMA_NAME_KEY, SCHEMA_VERSION_KEY): 

579 if k in metadata: 

580 del metadata[k] 

581 

582 return defects 

583 

584 @classmethod 

585 def readFits(cls, *args): 

586 """Read defect list from FITS table. 

587 

588 Parameters 

589 ---------- 

590 *args 

591 Arguments to be forwarded to 

592 `lsst.afw.table.BaseCatalog.writeFits`. 

593 

594 Returns 

595 ------- 

596 defects : `Defects` 

597 Defects read from a FITS table. 

598 """ 

599 table = lsst.afw.table.BaseCatalog.readFits(*args) 

600 return cls.fromTable(table) 

601 

602 @classmethod 

603 def readText(cls, filename): 

604 """Read defect list from standard format text table file. 

605 

606 Parameters 

607 ---------- 

608 filename : `str` 

609 Name of the file containing the defects definitions. 

610 

611 Returns 

612 ------- 

613 defects : `Defects` 

614 Defects read from a FITS table. 

615 """ 

616 table = astropy.table.Table.read(filename) 

617 

618 # Need to convert the Astropy table to afw table 

619 schema = lsst.afw.table.Schema() 

620 for colName in table.columns: 

621 schema.addField(colName, units=str(table[colName].unit), 

622 type=table[colName].dtype.type) 

623 

624 # Create AFW table that is required by fromTable() 

625 afwTable = lsst.afw.table.BaseCatalog(schema) 

626 

627 afwTable.resize(len(table)) 

628 for colName in table.columns: 

629 # String columns will fail -- currently we do not expect any 

630 afwTable[colName] = table[colName] 

631 

632 # Copy in the metadata from the astropy table 

633 metadata = PropertyList() 

634 for k, v in table.meta.items(): 

635 metadata[k] = v 

636 afwTable.setMetadata(metadata) 

637 

638 # Extract defect information from the table itself 

639 return cls.fromTable(afwTable) 

640 

641 @classmethod 

642 def readLsstDefectsFile(cls, filename): 

643 """Read defects information from a legacy LSST format text file. 

644 

645 Parameters 

646 ---------- 

647 filename : `str` 

648 Name of text file containing the defect information. 

649 

650 Returns 

651 ------- 

652 defects : `Defects` 

653 The defects. 

654 

655 Notes 

656 ----- 

657 These defect text files are used as the human readable definitions 

658 of defects in calibration data definition repositories. The format 

659 is to use four columns defined as follows: 

660 

661 x0 : `int` 

662 X coordinate of bottom left corner of box. 

663 y0 : `int` 

664 Y coordinate of bottom left corner of box. 

665 width : `int` 

666 X extent of the box. 

667 height : `int` 

668 Y extent of the box. 

669 

670 Files of this format were used historically to represent defects 

671 in simple text form. Use `Defects.readText` and `Defects.writeText` 

672 to use the more modern format. 

673 """ 

674 # Use loadtxt so that ValueError is thrown if the file contains a 

675 # non-integer value. genfromtxt converts bad values to -1. 

676 defect_array = np.loadtxt(filename, 

677 dtype=[("x0", "int"), ("y0", "int"), 

678 ("x_extent", "int"), ("y_extent", "int")]) 

679 

680 return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]), 

681 lsst.geom.Extent2I(row["x_extent"], row["y_extent"])) 

682 for row in defect_array) 

683 

684 @classmethod 

685 def fromFootprintList(cls, fpList): 

686 """Compute a defect list from a footprint list, optionally growing 

687 the footprints. 

688 

689 Parameters 

690 ---------- 

691 fpList : `list` of `lsst.afw.detection.Footprint` 

692 Footprint list to process. 

693 

694 Returns 

695 ------- 

696 defects : `Defects` 

697 List of defects. 

698 """ 

699 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp) 

700 for fp in fpList)) 

701 

702 @classmethod 

703 def fromMask(cls, maskedImage, maskName): 

704 """Compute a defect list from a specified mask plane. 

705 

706 Parameters 

707 ---------- 

708 maskedImage : `lsst.afw.image.MaskedImage` 

709 Image to process. 

710 maskName : `str` or `list` 

711 Mask plane name, or list of names to convert. 

712 

713 Returns 

714 ------- 

715 defects : `Defects` 

716 Defect list constructed from masked pixels. 

717 """ 

718 mask = maskedImage.getMask() 

719 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName), 

720 lsst.afw.detection.Threshold.BITMASK) 

721 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints() 

722 return cls.fromFootprintList(fpList)