Coverage for python/lsst/ip/isr/defects.py: 19%
258 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-20 10:19 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-20 10:19 +0000
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Support for image defects"""
23__all__ = ("Defects",)
25import logging
26import itertools
27import contextlib
28import numpy as np
29import math
30import numbers
31import astropy.table
33import lsst.geom
34import lsst.afw.table
35import lsst.afw.detection
36import lsst.afw.image
37import lsst.afw.geom
38from lsst.meas.algorithms import Defect
39from .calibType import IsrCalib
41log = logging.getLogger(__name__)
43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA"
44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION"
47class Defects(IsrCalib):
48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`.
50 Parameters
51 ----------
52 defectList : iterable, optional
53 Collections of defects to apply to the image. Can be an iterable of
54 `lsst.meas.algorithms.Defect` or `lsst.geom.BoxI`.
55 metadata : `lsst.daf.base.PropertyList`, optional
56 Metadata to associate with the defects. Will be copied and
57 overwrite existing metadata, if any. If not supplied the existing
58 metadata will be reset.
59 normalize_on_init : `bool`
60 If True, normalization is applied to the defects in ``defectList`` to
61 remove duplicates, eliminate overlaps, etc.
63 Notes
64 -----
65 Defects are stored within this collection in a "reduced" or "normalized"
66 form: rather than simply storing the bounding boxes which are added to the
67 collection, we eliminate overlaps and duplicates. This normalization
68 procedure may introduce overhead when adding many new defects; it may be
69 temporarily disabled using the `Defects.bulk_update` context manager if
70 necessary.
72 The attributes stored in this calibration are:
74 _defects : `list` [`lsst.meas.algorithms.Defect`]
75 The collection of Defect objects.
76 """
78 """The calibration type used for ingest."""
79 _OBSTYPE = "defects"
80 _SCHEMA = ''
81 _VERSION = 2.0
83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs):
84 self._defects = []
86 if defectList is not None:
87 self._bulk_update = True
88 for d in defectList:
89 self.append(d)
90 self._bulk_update = False
92 if normalize_on_init:
93 self._normalize()
95 super().__init__(**kwargs)
96 self.requiredAttributes.update(['_defects'])
98 def _check_value(self, value):
99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
100 or can be converted to one.
102 Parameters
103 ----------
104 value : `object`
105 Value to check.
107 Returns
108 -------
109 new : `~lsst.meas.algorithms.Defect`
110 Either the supplied value or a new object derived from it.
112 Raises
113 ------
114 ValueError
115 Raised if the supplied value can not be converted to
116 `~lsst.meas.algorithms.Defect`
117 """
118 if isinstance(value, Defect):
119 pass
120 elif isinstance(value, lsst.geom.BoxI):
121 value = Defect(value)
122 elif isinstance(value, lsst.geom.PointI):
123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
124 elif isinstance(value, lsst.afw.image.DefectBase):
125 value = Defect(value.getBBox())
126 else:
127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
128 return value
130 def __len__(self):
131 return len(self._defects)
133 def __getitem__(self, index):
134 return self._defects[index]
136 def __setitem__(self, index, value):
137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
138 """
139 self._defects[index] = self._check_value(value)
140 self._normalize()
142 def __iter__(self):
143 return iter(self._defects)
145 def __delitem__(self, index):
146 del self._defects[index]
148 def __eq__(self, other):
149 """Compare if two `Defects` are equal.
151 Two `Defects` are equal if their bounding boxes are equal and in
152 the same order. Metadata content is ignored.
153 """
154 super().__eq__(other)
156 if not isinstance(other, self.__class__):
157 return False
159 # checking the bboxes with zip() only works if same length
160 if len(self) != len(other):
161 return False
163 # Assume equal if bounding boxes are equal
164 for d1, d2 in zip(self, other):
165 if d1.getBBox() != d2.getBBox():
166 return False
168 return True
170 def __str__(self):
171 baseStr = super().__str__()
172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")"
174 def _normalize(self):
175 """Recalculate defect bounding boxes for efficiency.
177 Notes
178 -----
179 Ideally, this would generate the provably-minimal set of bounding
180 boxes necessary to represent the defects. At present, however, that
181 doesn't happen: see DM-24781. In the cases of substantial overlaps or
182 duplication, though, this will produce a much reduced set.
183 """
184 # In bulk-update mode, normalization is a no-op.
185 if self._bulk_update:
186 return
188 # If we have no defects, there is nothing to normalize.
189 if len(self) == 0:
190 return
192 # work out the minimum and maximum bounds from all defect regions.
193 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf')
194 for defect in self:
195 bbox = defect.getBBox()
196 minX = min(minX, bbox.getMinX())
197 minY = min(minY, bbox.getMinY())
198 maxX = max(maxX, bbox.getMaxX())
199 maxY = max(maxY, bbox.getMaxY())
201 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY),
202 lsst.geom.Point2I(maxX, maxY))
204 mask = lsst.afw.image.Mask(region)
205 self.maskPixels(mask, maskName="BAD")
206 self._defects = Defects.fromMask(mask, "BAD")._defects
208 @contextlib.contextmanager
209 def bulk_update(self):
210 """Temporarily suspend normalization of the defect list.
211 """
212 self._bulk_update = True
213 try:
214 yield
215 finally:
216 self._bulk_update = False
217 self._normalize()
219 def append(self, value):
220 self._defects.append(self._check_value(value))
221 self._normalize()
223 def insert(self, index, value):
224 self._defects.insert(index, self._check_value(value))
225 self._normalize()
227 def copy(self):
228 """Copy the defects to a new list, creating new defects from the
229 bounding boxes.
231 Returns
232 -------
233 new : `Defects`
234 New list with new `Defect` entries.
236 Notes
237 -----
238 This is not a shallow copy in that new `Defect` instances are
239 created from the original bounding boxes. It's also not a deep
240 copy since the bounding boxes are not recreated.
241 """
242 return self.__class__(d.getBBox() for d in self)
244 def transpose(self):
245 """Make a transposed copy of this defect list.
247 Returns
248 -------
249 retDefectList : `Defects`
250 Transposed list of defects.
251 """
252 retDefectList = self.__class__()
253 for defect in self:
254 bbox = defect.getBBox()
255 dimensions = bbox.getDimensions()
256 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
257 lsst.geom.Extent2I(dimensions[1], dimensions[0]))
258 retDefectList.append(nbbox)
259 return retDefectList
261 def maskPixels(self, mask, maskName="BAD"):
262 """Set mask plane based on these defects.
264 Parameters
265 ----------
266 maskedImage : `lsst.afw.image.MaskedImage` or `lsst.afw.image.Mask`
267 Image to process. Only the mask plane is updated.
268 maskName : str, optional
269 Mask plane name to use.
270 """
271 # mask bad pixels
272 if hasattr(mask, "getMask"):
273 mask = mask.getMask()
274 bitmask = mask.getPlaneBitMask(maskName)
275 for defect in self:
276 bbox = defect.getBBox()
277 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
279 def toFitsRegionTable(self):
280 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the
281 FITS region standard.
283 Returns
284 -------
285 table : `lsst.afw.table.BaseCatalog`
286 Defects in tabular form.
288 Notes
289 -----
290 The table created uses the
291 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
292 definition tabular format. The ``X`` and ``Y`` coordinates are
293 converted to FITS Physical coordinates that have origin pixel (1, 1)
294 rather than the (0, 0) used in LSST software.
295 """
296 self.updateMetadata()
297 nrows = len(self._defects)
299 if nrows:
300 # Adding entire columns is more efficient than adding
301 # each element separately
302 xCol = []
303 yCol = []
304 rCol = []
305 shapes = []
306 for i, defect in enumerate(self._defects):
307 box = defect.getBBox()
308 center = box.getCenter()
309 # Correct for the FITS 1-based offset
310 xCol.append(center.getX() + 1.0)
311 yCol.append(center.getY() + 1.0)
313 width = box.width
314 height = box.height
316 if width == 1 and height == 1:
317 # Call this a point
318 shapeType = "POINT"
319 else:
320 shapeType = "BOX"
322 # Strings have to be added per row
323 shapes.append(shapeType)
325 rCol.append(np.array([width, height], dtype=np.float64))
327 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes,
328 'R': rCol, 'ROTANG': np.zeros(nrows),
329 'COMPONENT': np.arange(nrows)})
330 table.meta = self.getMetadata().toDict()
331 return table
333 @classmethod
334 def fromDict(cls, dictionary):
335 """Construct a calibration from a dictionary of properties.
337 Must be implemented by the specific calibration subclasses.
339 Parameters
340 ----------
341 dictionary : `dict`
342 Dictionary of properties.
344 Returns
345 -------
346 calib : `lsst.ip.isr.CalibType`
347 Constructed calibration.
349 Raises
350 ------
351 RuntimeError
352 Raised if the supplied dictionary is for a different
353 calibration.
354 """
355 calib = cls()
357 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
358 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
359 f"found {dictionary['metadata']['OBSTYPE']}")
361 calib.setMetadata(dictionary['metadata'])
362 calib.calibInfoFromDict(dictionary)
364 xCol = dictionary['x0']
365 yCol = dictionary['y0']
366 widthCol = dictionary['width']
367 heightCol = dictionary['height']
369 with calib.bulk_update:
370 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol):
371 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0),
372 lsst.geom.Extent2I(width, height)))
373 return calib
375 def toDict(self):
376 """Return a dictionary containing the calibration properties.
378 The dictionary should be able to be round-tripped through
379 `fromDict`.
381 Returns
382 -------
383 dictionary : `dict`
384 Dictionary of properties.
385 """
386 self.updateMetadata()
388 outDict = {}
389 metadata = self.getMetadata()
390 outDict['metadata'] = metadata
392 xCol = []
393 yCol = []
394 widthCol = []
395 heightCol = []
397 nrows = len(self._defects)
398 if nrows:
399 for defect in self._defects:
400 box = defect.getBBox()
401 xCol.append(box.getBeginX())
402 yCol.append(box.getBeginY())
403 widthCol.append(box.getWidth())
404 heightCol.append(box.getHeight())
406 outDict['x0'] = xCol
407 outDict['y0'] = yCol
408 outDict['width'] = widthCol
409 outDict['height'] = heightCol
411 return outDict
413 def toTable(self):
414 """Convert defects to a simple table form that we use to write
415 to text files.
417 Returns
418 -------
419 table : `lsst.afw.table.BaseCatalog`
420 Defects in simple tabular form.
422 Notes
423 -----
424 These defect tables are used as the human readable definitions
425 of defects in calibration data definition repositories. The format
426 is to use four columns defined as follows:
428 x0 : `int`
429 X coordinate of bottom left corner of box.
430 y0 : `int`
431 Y coordinate of bottom left corner of box.
432 width : `int`
433 X extent of the box.
434 height : `int`
435 Y extent of the box.
436 """
437 tableList = []
438 self.updateMetadata()
440 xCol = []
441 yCol = []
442 widthCol = []
443 heightCol = []
445 nrows = len(self._defects)
446 if nrows:
447 for defect in self._defects:
448 box = defect.getBBox()
449 xCol.append(box.getBeginX())
450 yCol.append(box.getBeginY())
451 widthCol.append(box.getWidth())
452 heightCol.append(box.getHeight())
454 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol})
455 inMeta = self.getMetadata().toDict()
456 outMeta = {k: v for k, v in inMeta.items() if v is not None}
457 catalog.meta = outMeta
458 tableList.append(catalog)
460 return tableList
462 @staticmethod
463 def _get_values(values, n=1):
464 """Retrieve N values from the supplied values.
466 Parameters
467 ----------
468 values : `numbers.Number` or `list` or `np.array`
469 Input values.
470 n : `int`
471 Number of values to retrieve.
473 Returns
474 -------
475 vals : `list` or `np.array` or `numbers.Number`
476 Single value from supplied list if ``n`` is 1, or `list`
477 containing first ``n`` values from supplied values.
479 Notes
480 -----
481 Some supplied tables have vectors in some columns that can also
482 be scalars. This method can be used to get the first number as
483 a scalar or the first N items from a vector as a vector.
484 """
485 if n == 1:
486 if isinstance(values, numbers.Number):
487 return values
488 else:
489 return values[0]
491 return values[:n]
493 @classmethod
494 def fromTable(cls, tableList, normalize_on_init=True):
495 """Construct a `Defects` from the contents of a
496 `~lsst.afw.table.BaseCatalog`.
498 Parameters
499 ----------
500 table : `lsst.afw.table.BaseCatalog`
501 Table with one row per defect.
502 normalize_on_init : `bool`, optional
503 If `True`, normalization is applied to the defects listed in the
504 table to remove duplicates, eliminate overlaps, etc. Otherwise
505 the defects in the returned object exactly match those in the
506 table.
508 Returns
509 -------
510 defects : `Defects`
511 A `Defects` list.
513 Notes
514 -----
515 Two table formats are recognized. The first is the
516 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
517 definition tabular format written by `toFitsRegionTable` where the
518 pixel origin is corrected from FITS 1-based to a 0-based origin.
519 The second is the legacy defects format using columns ``x0``, ``y0``
520 (bottom left hand pixel of box in 0-based coordinates), ``width``
521 and ``height``.
523 The FITS standard regions can only read BOX, POINT, or ROTBOX with
524 a zero degree rotation.
525 """
526 table = tableList[0]
527 defectList = []
529 schema = table.columns
530 # Check schema to see which definitions we have
531 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
532 # This is a FITS region style table
533 isFitsRegion = True
534 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
535 # This is a classic LSST-style defect table
536 isFitsRegion = False
537 else:
538 raise ValueError("Unsupported schema for defects extraction")
540 for record in table:
541 if isFitsRegion:
542 # Coordinates can be arrays (some shapes in the standard
543 # require this)
544 # Correct for FITS 1-based origin
545 xcen = cls._get_values(record['X']) - 1.0
546 ycen = cls._get_values(record['Y']) - 1.0
547 shape = record['SHAPE'].upper().rstrip()
548 if shape == "BOX":
549 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
550 lsst.geom.Extent2I(cls._get_values(record['R'],
551 n=2)))
552 elif shape == "POINT":
553 # Handle the case where we have an externally created
554 # FITS file.
555 box = lsst.geom.Point2I(xcen, ycen)
556 elif shape == "ROTBOX":
557 # Astropy regions always writes ROTBOX
558 rotang = cls._get_values(record['ROTANG'])
559 # We can support 0 or 90 deg
560 if math.isclose(rotang % 90.0, 0.0):
561 # Two values required
562 r = cls._get_values(record['R'], n=2)
563 if math.isclose(rotang % 180.0, 0.0):
564 width = r[0]
565 height = r[1]
566 else:
567 width = r[1]
568 height = r[0]
569 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
570 lsst.geom.Extent2I(width, height))
571 else:
572 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
573 continue
574 else:
575 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
576 continue
578 else:
579 # This is a classic LSST-style defect table
580 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']),
581 lsst.geom.Extent2I(record['width'], record['height']))
583 defectList.append(box)
585 defects = cls(defectList, normalize_on_init=normalize_on_init)
586 newMeta = dict(table.meta)
587 defects.updateMetadata(setCalibInfo=True, **newMeta)
589 return defects
591 @classmethod
592 def readLsstDefectsFile(cls, filename, normalize_on_init=False):
593 """Read defects information from a legacy LSST format text file.
595 Parameters
596 ----------
597 filename : `str`
598 Name of text file containing the defect information.
600 normalize_on_init : `bool`, optional
601 If `True`, normalization is applied to the defects listed in the
602 table to remove duplicates, eliminate overlaps, etc. Otherwise
603 the defects in the returned object exactly match those in the
604 table.
606 Returns
607 -------
608 defects : `Defects`
609 The defects.
611 Notes
612 -----
613 These defect text files are used as the human readable definitions
614 of defects in calibration data definition repositories. The format
615 is to use four columns defined as follows:
617 x0 : `int`
618 X coordinate of bottom left corner of box.
619 y0 : `int`
620 Y coordinate of bottom left corner of box.
621 width : `int`
622 X extent of the box.
623 height : `int`
624 Y extent of the box.
626 Files of this format were used historically to represent defects
627 in simple text form. Use `Defects.readText` and `Defects.writeText`
628 to use the more modern format.
629 """
630 # Use loadtxt so that ValueError is thrown if the file contains a
631 # non-integer value. genfromtxt converts bad values to -1.
632 defect_array = np.loadtxt(filename,
633 dtype=[("x0", "int"), ("y0", "int"),
634 ("x_extent", "int"), ("y_extent", "int")])
636 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
637 lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
638 for row in defect_array)
640 return cls(defects, normalize_on_init=normalize_on_init)
642 @classmethod
643 def fromFootprintList(cls, fpList):
644 """Compute a defect list from a footprint list, optionally growing
645 the footprints.
647 Parameters
648 ----------
649 fpList : `list` of `lsst.afw.detection.Footprint`
650 Footprint list to process.
652 Returns
653 -------
654 defects : `Defects`
655 List of defects.
656 """
657 # normalize_on_init is set to False to avoid recursively calling
658 # fromMask/fromFootprintList in Defects.__init__.
659 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
660 for fp in fpList), normalize_on_init=False)
662 @classmethod
663 def fromMask(cls, mask, maskName):
664 """Compute a defect list from a specified mask plane.
666 Parameters
667 ----------
668 mask : `lsst.afw.image.Mask` or `lsst.afw.image.MaskedImage`
669 Image to process.
670 maskName : `str` or `list`
671 Mask plane name, or list of names to convert.
673 Returns
674 -------
675 defects : `Defects`
676 Defect list constructed from masked pixels.
677 """
678 if hasattr(mask, "getMask"):
679 mask = mask.getMask()
680 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
681 lsst.afw.detection.Threshold.BITMASK)
682 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
683 return cls.fromFootprintList(fpList)