Coverage for python/lsst/ip/isr/defects.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Support for image defects"""
23__all__ = ("Defects",)
25import logging
26import itertools
27import contextlib
28import numpy as np
29import math
30import numbers
31import astropy.table
33import lsst.geom
34import lsst.afw.table
35import lsst.afw.detection
36import lsst.afw.image
37import lsst.afw.geom
38from lsst.meas.algorithms import Defect
39from .calibType import IsrCalib
41log = logging.getLogger(__name__)
43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA"
44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION"
47class Defects(IsrCalib):
48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`.
50 Parameters
51 ----------
52 defectList : iterable of `lsst.meas.algorithms.Defect`
53 or `lsst.geom.BoxI`, optional
54 Collections of defects to apply to the image.
55 metadata : `lsst.daf.base.PropertyList`, optional
56 Metadata to associate with the defects. Will be copied and
57 overwrite existing metadata, if any. If not supplied the existing
58 metadata will be reset.
59 normalize_on_init : `bool`
60 If True, normalization is applied to the defects in ``defectList`` to
61 remove duplicates, eliminate overlaps, etc.
63 Notes
64 -----
65 Defects are stored within this collection in a "reduced" or "normalized"
66 form: rather than simply storing the bounding boxes which are added to the
67 collection, we eliminate overlaps and duplicates. This normalization
68 procedure may introduce overhead when adding many new defects; it may be
69 temporarily disabled using the `Defects.bulk_update` context manager if
70 necessary.
72 The attributes stored in this calibration are:
74 _defects : `list` [`lsst.meas.algorithms.Defect`]
75 The collection of Defect objects.
76 """
78 """The calibration type used for ingest."""
79 _OBSTYPE = "defects"
80 _SCHEMA = ''
81 _VERSION = 2.0
83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs):
84 self._defects = []
86 if defectList is not None:
87 self._bulk_update = True
88 for d in defectList:
89 self.append(d)
90 self._bulk_update = False
92 if normalize_on_init:
93 self._normalize()
95 super().__init__(**kwargs)
96 self.requiredAttributes.update(['_defects'])
98 def _check_value(self, value):
99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
100 or can be converted to one.
102 Parameters
103 ----------
104 value : `object`
105 Value to check.
107 Returns
108 -------
109 new : `~lsst.meas.algorithms.Defect`
110 Either the supplied value or a new object derived from it.
112 Raises
113 ------
114 ValueError
115 Raised if the supplied value can not be converted to
116 `~lsst.meas.algorithms.Defect`
117 """
118 if isinstance(value, Defect):
119 pass
120 elif isinstance(value, lsst.geom.BoxI):
121 value = Defect(value)
122 elif isinstance(value, lsst.geom.PointI):
123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
124 elif isinstance(value, lsst.afw.image.DefectBase):
125 value = Defect(value.getBBox())
126 else:
127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
128 return value
130 def __len__(self):
131 return len(self._defects)
133 def __getitem__(self, index):
134 return self._defects[index]
136 def __setitem__(self, index, value):
137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
138 """
139 self._defects[index] = self._check_value(value)
140 self._normalize()
142 def __iter__(self):
143 return iter(self._defects)
145 def __delitem__(self, index):
146 del self._defects[index]
148 def __eq__(self, other):
149 """Compare if two `Defects` are equal.
151 Two `Defects` are equal if their bounding boxes are equal and in
152 the same order. Metadata content is ignored.
153 """
154 super().__eq__(other)
156 if not isinstance(other, self.__class__):
157 return False
159 # checking the bboxes with zip() only works if same length
160 if len(self) != len(other):
161 return False
163 # Assume equal if bounding boxes are equal
164 for d1, d2 in zip(self, other):
165 if d1.getBBox() != d2.getBBox():
166 return False
168 return True
170 def __str__(self):
171 baseStr = super().__str__(self)
172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")"
174 def _normalize(self):
175 """Recalculate defect bounding boxes for efficiency.
177 Notes
178 -----
179 Ideally, this would generate the provably-minimal set of bounding
180 boxes necessary to represent the defects. At present, however, that
181 doesn't happen: see DM-24781. In the cases of substantial overlaps or
182 duplication, though, this will produce a much reduced set.
183 """
184 # In bulk-update mode, normalization is a no-op.
185 if self._bulk_update:
186 return
188 # work out the minimum and maximum bounds from all defect regions.
189 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf')
190 for defect in self:
191 bbox = defect.getBBox()
192 minX = min(minX, bbox.getMinX())
193 minY = min(minY, bbox.getMinY())
194 maxX = max(maxX, bbox.getMaxX())
195 maxY = max(maxY, bbox.getMaxY())
197 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY),
198 lsst.geom.Point2I(maxX, maxY))
200 mi = lsst.afw.image.MaskedImageF(region)
201 self.maskPixels(mi, maskName="BAD")
202 self._defects = Defects.fromMask(mi, "BAD")._defects
204 @contextlib.contextmanager
205 def bulk_update(self):
206 """Temporarily suspend normalization of the defect list.
207 """
208 self._bulk_update = True
209 try:
210 yield
211 finally:
212 self._bulk_update = False
213 self._normalize()
215 def append(self, value):
216 self._defects.append(self._check_value(value))
217 self._normalize()
219 def insert(self, index, value):
220 self._defects.insert(index, self._check_value(value))
221 self._normalize()
223 def copy(self):
224 """Copy the defects to a new list, creating new defects from the
225 bounding boxes.
227 Returns
228 -------
229 new : `Defects`
230 New list with new `Defect` entries.
232 Notes
233 -----
234 This is not a shallow copy in that new `Defect` instances are
235 created from the original bounding boxes. It's also not a deep
236 copy since the bounding boxes are not recreated.
237 """
238 return self.__class__(d.getBBox() for d in self)
240 def transpose(self):
241 """Make a transposed copy of this defect list.
243 Returns
244 -------
245 retDefectList : `Defects`
246 Transposed list of defects.
247 """
248 retDefectList = self.__class__()
249 for defect in self:
250 bbox = defect.getBBox()
251 dimensions = bbox.getDimensions()
252 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
253 lsst.geom.Extent2I(dimensions[1], dimensions[0]))
254 retDefectList.append(nbbox)
255 return retDefectList
257 def maskPixels(self, maskedImage, maskName="BAD"):
258 """Set mask plane based on these defects.
260 Parameters
261 ----------
262 maskedImage : `lsst.afw.image.MaskedImage`
263 Image to process. Only the mask plane is updated.
264 maskName : str, optional
265 Mask plane name to use.
266 """
267 # mask bad pixels
268 mask = maskedImage.getMask()
269 bitmask = mask.getPlaneBitMask(maskName)
270 for defect in self:
271 bbox = defect.getBBox()
272 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
274 def toFitsRegionTable(self):
275 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the
276 FITS region standard.
278 Returns
279 -------
280 table : `lsst.afw.table.BaseCatalog`
281 Defects in tabular form.
283 Notes
284 -----
285 The table created uses the
286 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
287 definition tabular format. The ``X`` and ``Y`` coordinates are
288 converted to FITS Physical coordinates that have origin pixel (1, 1)
289 rather than the (0, 0) used in LSST software.
290 """
291 self.updateMetadata()
292 nrows = len(self._defects)
294 if nrows:
295 # Adding entire columns is more efficient than adding
296 # each element separately
297 xCol = []
298 yCol = []
299 rCol = []
300 shapes = []
301 for i, defect in enumerate(self._defects):
302 box = defect.getBBox()
303 center = box.getCenter()
304 # Correct for the FITS 1-based offset
305 xCol.append(center.getX() + 1.0)
306 yCol.append(center.getY() + 1.0)
308 width = box.width
309 height = box.height
311 if width == 1 and height == 1:
312 # Call this a point
313 shapeType = "POINT"
314 else:
315 shapeType = "BOX"
317 # Strings have to be added per row
318 shapes.append(shapeType)
320 rCol.append(np.array([width, height], dtype=np.float64))
322 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes,
323 'R': rCol, 'ROTANG': np.zeros(nrows),
324 'COMPONENT': np.arange(nrows)})
325 table.meta = self.getMetadata().toDict()
326 return table
328 @classmethod
329 def fromDict(cls, dictionary):
330 """Construct a calibration from a dictionary of properties.
332 Must be implemented by the specific calibration subclasses.
334 Parameters
335 ----------
336 dictionary : `dict`
337 Dictionary of properties.
339 Returns
340 -------
341 calib : `lsst.ip.isr.CalibType`
342 Constructed calibration.
344 Raises
345 ------
346 RuntimeError :
347 Raised if the supplied dictionary is for a different
348 calibration.
349 """
350 calib = cls()
352 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
353 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
354 f"found {dictionary['metadata']['OBSTYPE']}")
356 calib.setMetadata(dictionary['metadata'])
357 calib.calibInfoFromDict(dictionary)
359 xCol = dictionary['x0']
360 yCol = dictionary['y0']
361 widthCol = dictionary['width']
362 heightCol = dictionary['height']
364 with calib.bulk_update:
365 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol):
366 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0),
367 lsst.geom.Extent2I(width, height)))
368 return calib
370 def toDict(self):
371 """Return a dictionary containing the calibration properties.
373 The dictionary should be able to be round-tripped through
374 `fromDict`.
376 Returns
377 -------
378 dictionary : `dict`
379 Dictionary of properties.
380 """
381 self.updateMetadata()
383 outDict = {}
384 metadata = self.getMetadata()
385 outDict['metadata'] = metadata
387 xCol = []
388 yCol = []
389 widthCol = []
390 heightCol = []
392 nrows = len(self._defects)
393 if nrows:
394 for defect in self._defects:
395 box = defect.getBBox()
396 xCol.append(box.getBeginX())
397 yCol.append(box.getBeginY())
398 widthCol.append(box.getWidth())
399 heightCol.append(box.getHeight())
401 outDict['x0'] = xCol
402 outDict['y0'] = yCol
403 outDict['width'] = widthCol
404 outDict['height'] = heightCol
406 return outDict
408 def toTable(self):
409 """Convert defects to a simple table form that we use to write
410 to text files.
412 Returns
413 -------
414 table : `lsst.afw.table.BaseCatalog`
415 Defects in simple tabular form.
417 Notes
418 -----
419 These defect tables are used as the human readable definitions
420 of defects in calibration data definition repositories. The format
421 is to use four columns defined as follows:
423 x0 : `int`
424 X coordinate of bottom left corner of box.
425 y0 : `int`
426 Y coordinate of bottom left corner of box.
427 width : `int`
428 X extent of the box.
429 height : `int`
430 Y extent of the box.
431 """
432 tableList = []
433 self.updateMetadata()
435 xCol = []
436 yCol = []
437 widthCol = []
438 heightCol = []
440 nrows = len(self._defects)
441 if nrows:
442 for defect in self._defects:
443 box = defect.getBBox()
444 xCol.append(box.getBeginX())
445 yCol.append(box.getBeginY())
446 widthCol.append(box.getWidth())
447 heightCol.append(box.getHeight())
449 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol})
450 inMeta = self.getMetadata().toDict()
451 outMeta = {k: v for k, v in inMeta.items() if v is not None}
452 catalog.meta = outMeta
453 tableList.append(catalog)
455 return tableList
457 @staticmethod
458 def _get_values(values, n=1):
459 """Retrieve N values from the supplied values.
461 Parameters
462 ----------
463 values : `numbers.Number` or `list` or `np.array`
464 Input values.
465 n : `int`
466 Number of values to retrieve.
468 Returns
469 -------
470 vals : `list` or `np.array` or `numbers.Number`
471 Single value from supplied list if ``n`` is 1, or `list`
472 containing first ``n`` values from supplied values.
474 Notes
475 -----
476 Some supplied tables have vectors in some columns that can also
477 be scalars. This method can be used to get the first number as
478 a scalar or the first N items from a vector as a vector.
479 """
480 if n == 1:
481 if isinstance(values, numbers.Number):
482 return values
483 else:
484 return values[0]
486 return values[:n]
488 @classmethod
489 def fromTable(cls, tableList, normalize_on_init=True):
490 """Construct a `Defects` from the contents of a
491 `~lsst.afw.table.BaseCatalog`.
493 Parameters
494 ----------
495 table : `lsst.afw.table.BaseCatalog`
496 Table with one row per defect.
497 normalize_on_init : `bool`, optional
498 If `True`, normalization is applied to the defects listed in the
499 table to remove duplicates, eliminate overlaps, etc. Otherwise
500 the defects in the returned object exactly match those in the
501 table.
503 Returns
504 -------
505 defects : `Defects`
506 A `Defects` list.
508 Notes
509 -----
510 Two table formats are recognized. The first is the
511 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
512 definition tabular format written by `toFitsRegionTable` where the
513 pixel origin is corrected from FITS 1-based to a 0-based origin.
514 The second is the legacy defects format using columns ``x0``, ``y0``
515 (bottom left hand pixel of box in 0-based coordinates), ``width``
516 and ``height``.
518 The FITS standard regions can only read BOX, POINT, or ROTBOX with
519 a zero degree rotation.
520 """
521 table = tableList[0]
522 defectList = []
524 schema = table.columns
525 # Check schema to see which definitions we have
526 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
527 # This is a FITS region style table
528 isFitsRegion = True
529 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
530 # This is a classic LSST-style defect table
531 isFitsRegion = False
532 else:
533 raise ValueError("Unsupported schema for defects extraction")
535 for record in table:
536 if isFitsRegion:
537 # Coordinates can be arrays (some shapes in the standard
538 # require this)
539 # Correct for FITS 1-based origin
540 xcen = cls._get_values(record['X']) - 1.0
541 ycen = cls._get_values(record['Y']) - 1.0
542 shape = record['SHAPE'].upper().rstrip()
543 if shape == "BOX":
544 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
545 lsst.geom.Extent2I(cls._get_values(record['R'],
546 n=2)))
547 elif shape == "POINT":
548 # Handle the case where we have an externally created
549 # FITS file.
550 box = lsst.geom.Point2I(xcen, ycen)
551 elif shape == "ROTBOX":
552 # Astropy regions always writes ROTBOX
553 rotang = cls._get_values(record['ROTANG'])
554 # We can support 0 or 90 deg
555 if math.isclose(rotang % 90.0, 0.0):
556 # Two values required
557 r = cls._get_values(record['R'], n=2)
558 if math.isclose(rotang % 180.0, 0.0):
559 width = r[0]
560 height = r[1]
561 else:
562 width = r[1]
563 height = r[0]
564 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
565 lsst.geom.Extent2I(width, height))
566 else:
567 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
568 continue
569 else:
570 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
571 continue
573 else:
574 # This is a classic LSST-style defect table
575 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']),
576 lsst.geom.Extent2I(record['width'], record['height']))
578 defectList.append(box)
580 defects = cls(defectList, normalize_on_init=normalize_on_init)
581 newMeta = dict(table.meta)
582 defects.updateMetadata(setCalibInfo=True, **newMeta)
584 return defects
586 @classmethod
587 def readLsstDefectsFile(cls, filename, normalize_on_init=False):
588 """Read defects information from a legacy LSST format text file.
590 Parameters
591 ----------
592 filename : `str`
593 Name of text file containing the defect information.
595 normalize_on_init : `bool`, optional
596 If `True`, normalization is applied to the defects listed in the
597 table to remove duplicates, eliminate overlaps, etc. Otherwise
598 the defects in the returned object exactly match those in the
599 table.
601 Returns
602 -------
603 defects : `Defects`
604 The defects.
606 Notes
607 -----
608 These defect text files are used as the human readable definitions
609 of defects in calibration data definition repositories. The format
610 is to use four columns defined as follows:
612 x0 : `int`
613 X coordinate of bottom left corner of box.
614 y0 : `int`
615 Y coordinate of bottom left corner of box.
616 width : `int`
617 X extent of the box.
618 height : `int`
619 Y extent of the box.
621 Files of this format were used historically to represent defects
622 in simple text form. Use `Defects.readText` and `Defects.writeText`
623 to use the more modern format.
624 """
625 # Use loadtxt so that ValueError is thrown if the file contains a
626 # non-integer value. genfromtxt converts bad values to -1.
627 defect_array = np.loadtxt(filename,
628 dtype=[("x0", "int"), ("y0", "int"),
629 ("x_extent", "int"), ("y_extent", "int")])
631 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
632 lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
633 for row in defect_array)
635 return cls(defects, normalize_on_init=normalize_on_init)
637 @classmethod
638 def fromFootprintList(cls, fpList):
639 """Compute a defect list from a footprint list, optionally growing
640 the footprints.
642 Parameters
643 ----------
644 fpList : `list` of `lsst.afw.detection.Footprint`
645 Footprint list to process.
647 Returns
648 -------
649 defects : `Defects`
650 List of defects.
651 """
652 # normalize_on_init is set to False to avoid recursively calling
653 # fromMask/fromFootprintList in Defects.__init__.
654 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
655 for fp in fpList), normalize_on_init=False)
657 @classmethod
658 def fromMask(cls, maskedImage, maskName):
659 """Compute a defect list from a specified mask plane.
661 Parameters
662 ----------
663 maskedImage : `lsst.afw.image.MaskedImage`
664 Image to process.
665 maskName : `str` or `list`
666 Mask plane name, or list of names to convert.
668 Returns
669 -------
670 defects : `Defects`
671 Defect list constructed from masked pixels.
672 """
673 mask = maskedImage.getMask()
674 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
675 lsst.afw.detection.Threshold.BITMASK)
676 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
677 return cls.fromFootprintList(fpList)