Coverage for python/lsst/ip/isr/defects.py: 16%
256 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-31 02:53 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-31 02:53 -0800
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21"""Support for image defects"""
23__all__ = ("Defects",)
25import logging
26import itertools
27import contextlib
28import numpy as np
29import math
30import numbers
31import astropy.table
33import lsst.geom
34import lsst.afw.table
35import lsst.afw.detection
36import lsst.afw.image
37import lsst.afw.geom
38from lsst.meas.algorithms import Defect
39from .calibType import IsrCalib
41log = logging.getLogger(__name__)
43SCHEMA_NAME_KEY = "DEFECTS_SCHEMA"
44SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION"
47class Defects(IsrCalib):
48 """Calibration handler for collections of `lsst.meas.algorithms.Defect`.
50 Parameters
51 ----------
52 defectList : iterable, optional
53 Collections of defects to apply to the image. Can be an iterable of
54 `lsst.meas.algorithms.Defect` or `lsst.geom.BoxI`.
55 metadata : `lsst.daf.base.PropertyList`, optional
56 Metadata to associate with the defects. Will be copied and
57 overwrite existing metadata, if any. If not supplied the existing
58 metadata will be reset.
59 normalize_on_init : `bool`
60 If True, normalization is applied to the defects in ``defectList`` to
61 remove duplicates, eliminate overlaps, etc.
63 Notes
64 -----
65 Defects are stored within this collection in a "reduced" or "normalized"
66 form: rather than simply storing the bounding boxes which are added to the
67 collection, we eliminate overlaps and duplicates. This normalization
68 procedure may introduce overhead when adding many new defects; it may be
69 temporarily disabled using the `Defects.bulk_update` context manager if
70 necessary.
72 The attributes stored in this calibration are:
74 _defects : `list` [`lsst.meas.algorithms.Defect`]
75 The collection of Defect objects.
76 """
78 """The calibration type used for ingest."""
79 _OBSTYPE = "defects"
80 _SCHEMA = ''
81 _VERSION = 2.0
83 def __init__(self, defectList=None, metadata=None, *, normalize_on_init=True, **kwargs):
84 self._defects = []
86 if defectList is not None:
87 self._bulk_update = True
88 for d in defectList:
89 self.append(d)
90 self._bulk_update = False
92 if normalize_on_init:
93 self._normalize()
95 super().__init__(**kwargs)
96 self.requiredAttributes.update(['_defects'])
98 def _check_value(self, value):
99 """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
100 or can be converted to one.
102 Parameters
103 ----------
104 value : `object`
105 Value to check.
107 Returns
108 -------
109 new : `~lsst.meas.algorithms.Defect`
110 Either the supplied value or a new object derived from it.
112 Raises
113 ------
114 ValueError
115 Raised if the supplied value can not be converted to
116 `~lsst.meas.algorithms.Defect`
117 """
118 if isinstance(value, Defect):
119 pass
120 elif isinstance(value, lsst.geom.BoxI):
121 value = Defect(value)
122 elif isinstance(value, lsst.geom.PointI):
123 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
124 elif isinstance(value, lsst.afw.image.DefectBase):
125 value = Defect(value.getBBox())
126 else:
127 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
128 return value
130 def __len__(self):
131 return len(self._defects)
133 def __getitem__(self, index):
134 return self._defects[index]
136 def __setitem__(self, index, value):
137 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
138 """
139 self._defects[index] = self._check_value(value)
140 self._normalize()
142 def __iter__(self):
143 return iter(self._defects)
145 def __delitem__(self, index):
146 del self._defects[index]
148 def __eq__(self, other):
149 """Compare if two `Defects` are equal.
151 Two `Defects` are equal if their bounding boxes are equal and in
152 the same order. Metadata content is ignored.
153 """
154 super().__eq__(other)
156 if not isinstance(other, self.__class__):
157 return False
159 # checking the bboxes with zip() only works if same length
160 if len(self) != len(other):
161 return False
163 # Assume equal if bounding boxes are equal
164 for d1, d2 in zip(self, other):
165 if d1.getBBox() != d2.getBBox():
166 return False
168 return True
170 def __str__(self):
171 baseStr = super().__str__()
172 return baseStr + ",".join(str(d.getBBox()) for d in self) + ")"
174 def _normalize(self):
175 """Recalculate defect bounding boxes for efficiency.
177 Notes
178 -----
179 Ideally, this would generate the provably-minimal set of bounding
180 boxes necessary to represent the defects. At present, however, that
181 doesn't happen: see DM-24781. In the cases of substantial overlaps or
182 duplication, though, this will produce a much reduced set.
183 """
184 # In bulk-update mode, normalization is a no-op.
185 if self._bulk_update:
186 return
188 # If we have no defects, there is nothing to normalize.
189 if len(self) == 0:
190 return
192 # work out the minimum and maximum bounds from all defect regions.
193 minX, minY, maxX, maxY = float('inf'), float('inf'), float('-inf'), float('-inf')
194 for defect in self:
195 bbox = defect.getBBox()
196 minX = min(minX, bbox.getMinX())
197 minY = min(minY, bbox.getMinY())
198 maxX = max(maxX, bbox.getMaxX())
199 maxY = max(maxY, bbox.getMaxY())
201 region = lsst.geom.Box2I(lsst.geom.Point2I(minX, minY),
202 lsst.geom.Point2I(maxX, maxY))
204 mi = lsst.afw.image.MaskedImageF(region)
205 self.maskPixels(mi, maskName="BAD")
206 self._defects = Defects.fromMask(mi, "BAD")._defects
208 @contextlib.contextmanager
209 def bulk_update(self):
210 """Temporarily suspend normalization of the defect list.
211 """
212 self._bulk_update = True
213 try:
214 yield
215 finally:
216 self._bulk_update = False
217 self._normalize()
219 def append(self, value):
220 self._defects.append(self._check_value(value))
221 self._normalize()
223 def insert(self, index, value):
224 self._defects.insert(index, self._check_value(value))
225 self._normalize()
227 def copy(self):
228 """Copy the defects to a new list, creating new defects from the
229 bounding boxes.
231 Returns
232 -------
233 new : `Defects`
234 New list with new `Defect` entries.
236 Notes
237 -----
238 This is not a shallow copy in that new `Defect` instances are
239 created from the original bounding boxes. It's also not a deep
240 copy since the bounding boxes are not recreated.
241 """
242 return self.__class__(d.getBBox() for d in self)
244 def transpose(self):
245 """Make a transposed copy of this defect list.
247 Returns
248 -------
249 retDefectList : `Defects`
250 Transposed list of defects.
251 """
252 retDefectList = self.__class__()
253 for defect in self:
254 bbox = defect.getBBox()
255 dimensions = bbox.getDimensions()
256 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
257 lsst.geom.Extent2I(dimensions[1], dimensions[0]))
258 retDefectList.append(nbbox)
259 return retDefectList
261 def maskPixels(self, maskedImage, maskName="BAD"):
262 """Set mask plane based on these defects.
264 Parameters
265 ----------
266 maskedImage : `lsst.afw.image.MaskedImage`
267 Image to process. Only the mask plane is updated.
268 maskName : str, optional
269 Mask plane name to use.
270 """
271 # mask bad pixels
272 mask = maskedImage.getMask()
273 bitmask = mask.getPlaneBitMask(maskName)
274 for defect in self:
275 bbox = defect.getBBox()
276 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
278 def toFitsRegionTable(self):
279 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the
280 FITS region standard.
282 Returns
283 -------
284 table : `lsst.afw.table.BaseCatalog`
285 Defects in tabular form.
287 Notes
288 -----
289 The table created uses the
290 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
291 definition tabular format. The ``X`` and ``Y`` coordinates are
292 converted to FITS Physical coordinates that have origin pixel (1, 1)
293 rather than the (0, 0) used in LSST software.
294 """
295 self.updateMetadata()
296 nrows = len(self._defects)
298 if nrows:
299 # Adding entire columns is more efficient than adding
300 # each element separately
301 xCol = []
302 yCol = []
303 rCol = []
304 shapes = []
305 for i, defect in enumerate(self._defects):
306 box = defect.getBBox()
307 center = box.getCenter()
308 # Correct for the FITS 1-based offset
309 xCol.append(center.getX() + 1.0)
310 yCol.append(center.getY() + 1.0)
312 width = box.width
313 height = box.height
315 if width == 1 and height == 1:
316 # Call this a point
317 shapeType = "POINT"
318 else:
319 shapeType = "BOX"
321 # Strings have to be added per row
322 shapes.append(shapeType)
324 rCol.append(np.array([width, height], dtype=np.float64))
326 table = astropy.table.Table({'X': xCol, 'Y': yCol, 'SHAPE': shapes,
327 'R': rCol, 'ROTANG': np.zeros(nrows),
328 'COMPONENT': np.arange(nrows)})
329 table.meta = self.getMetadata().toDict()
330 return table
332 @classmethod
333 def fromDict(cls, dictionary):
334 """Construct a calibration from a dictionary of properties.
336 Must be implemented by the specific calibration subclasses.
338 Parameters
339 ----------
340 dictionary : `dict`
341 Dictionary of properties.
343 Returns
344 -------
345 calib : `lsst.ip.isr.CalibType`
346 Constructed calibration.
348 Raises
349 ------
350 RuntimeError
351 Raised if the supplied dictionary is for a different
352 calibration.
353 """
354 calib = cls()
356 if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
357 raise RuntimeError(f"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
358 f"found {dictionary['metadata']['OBSTYPE']}")
360 calib.setMetadata(dictionary['metadata'])
361 calib.calibInfoFromDict(dictionary)
363 xCol = dictionary['x0']
364 yCol = dictionary['y0']
365 widthCol = dictionary['width']
366 heightCol = dictionary['height']
368 with calib.bulk_update:
369 for x0, y0, width, height in zip(xCol, yCol, widthCol, heightCol):
370 calib.append(lsst.geom.Box2I(lsst.geom.Point2I(x0, y0),
371 lsst.geom.Extent2I(width, height)))
372 return calib
374 def toDict(self):
375 """Return a dictionary containing the calibration properties.
377 The dictionary should be able to be round-tripped through
378 `fromDict`.
380 Returns
381 -------
382 dictionary : `dict`
383 Dictionary of properties.
384 """
385 self.updateMetadata()
387 outDict = {}
388 metadata = self.getMetadata()
389 outDict['metadata'] = metadata
391 xCol = []
392 yCol = []
393 widthCol = []
394 heightCol = []
396 nrows = len(self._defects)
397 if nrows:
398 for defect in self._defects:
399 box = defect.getBBox()
400 xCol.append(box.getBeginX())
401 yCol.append(box.getBeginY())
402 widthCol.append(box.getWidth())
403 heightCol.append(box.getHeight())
405 outDict['x0'] = xCol
406 outDict['y0'] = yCol
407 outDict['width'] = widthCol
408 outDict['height'] = heightCol
410 return outDict
412 def toTable(self):
413 """Convert defects to a simple table form that we use to write
414 to text files.
416 Returns
417 -------
418 table : `lsst.afw.table.BaseCatalog`
419 Defects in simple tabular form.
421 Notes
422 -----
423 These defect tables are used as the human readable definitions
424 of defects in calibration data definition repositories. The format
425 is to use four columns defined as follows:
427 x0 : `int`
428 X coordinate of bottom left corner of box.
429 y0 : `int`
430 Y coordinate of bottom left corner of box.
431 width : `int`
432 X extent of the box.
433 height : `int`
434 Y extent of the box.
435 """
436 tableList = []
437 self.updateMetadata()
439 xCol = []
440 yCol = []
441 widthCol = []
442 heightCol = []
444 nrows = len(self._defects)
445 if nrows:
446 for defect in self._defects:
447 box = defect.getBBox()
448 xCol.append(box.getBeginX())
449 yCol.append(box.getBeginY())
450 widthCol.append(box.getWidth())
451 heightCol.append(box.getHeight())
453 catalog = astropy.table.Table({'x0': xCol, 'y0': yCol, 'width': widthCol, 'height': heightCol})
454 inMeta = self.getMetadata().toDict()
455 outMeta = {k: v for k, v in inMeta.items() if v is not None}
456 catalog.meta = outMeta
457 tableList.append(catalog)
459 return tableList
461 @staticmethod
462 def _get_values(values, n=1):
463 """Retrieve N values from the supplied values.
465 Parameters
466 ----------
467 values : `numbers.Number` or `list` or `np.array`
468 Input values.
469 n : `int`
470 Number of values to retrieve.
472 Returns
473 -------
474 vals : `list` or `np.array` or `numbers.Number`
475 Single value from supplied list if ``n`` is 1, or `list`
476 containing first ``n`` values from supplied values.
478 Notes
479 -----
480 Some supplied tables have vectors in some columns that can also
481 be scalars. This method can be used to get the first number as
482 a scalar or the first N items from a vector as a vector.
483 """
484 if n == 1:
485 if isinstance(values, numbers.Number):
486 return values
487 else:
488 return values[0]
490 return values[:n]
492 @classmethod
493 def fromTable(cls, tableList, normalize_on_init=True):
494 """Construct a `Defects` from the contents of a
495 `~lsst.afw.table.BaseCatalog`.
497 Parameters
498 ----------
499 table : `lsst.afw.table.BaseCatalog`
500 Table with one row per defect.
501 normalize_on_init : `bool`, optional
502 If `True`, normalization is applied to the defects listed in the
503 table to remove duplicates, eliminate overlaps, etc. Otherwise
504 the defects in the returned object exactly match those in the
505 table.
507 Returns
508 -------
509 defects : `Defects`
510 A `Defects` list.
512 Notes
513 -----
514 Two table formats are recognized. The first is the
515 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
516 definition tabular format written by `toFitsRegionTable` where the
517 pixel origin is corrected from FITS 1-based to a 0-based origin.
518 The second is the legacy defects format using columns ``x0``, ``y0``
519 (bottom left hand pixel of box in 0-based coordinates), ``width``
520 and ``height``.
522 The FITS standard regions can only read BOX, POINT, or ROTBOX with
523 a zero degree rotation.
524 """
525 table = tableList[0]
526 defectList = []
528 schema = table.columns
529 # Check schema to see which definitions we have
530 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
531 # This is a FITS region style table
532 isFitsRegion = True
533 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
534 # This is a classic LSST-style defect table
535 isFitsRegion = False
536 else:
537 raise ValueError("Unsupported schema for defects extraction")
539 for record in table:
540 if isFitsRegion:
541 # Coordinates can be arrays (some shapes in the standard
542 # require this)
543 # Correct for FITS 1-based origin
544 xcen = cls._get_values(record['X']) - 1.0
545 ycen = cls._get_values(record['Y']) - 1.0
546 shape = record['SHAPE'].upper().rstrip()
547 if shape == "BOX":
548 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
549 lsst.geom.Extent2I(cls._get_values(record['R'],
550 n=2)))
551 elif shape == "POINT":
552 # Handle the case where we have an externally created
553 # FITS file.
554 box = lsst.geom.Point2I(xcen, ycen)
555 elif shape == "ROTBOX":
556 # Astropy regions always writes ROTBOX
557 rotang = cls._get_values(record['ROTANG'])
558 # We can support 0 or 90 deg
559 if math.isclose(rotang % 90.0, 0.0):
560 # Two values required
561 r = cls._get_values(record['R'], n=2)
562 if math.isclose(rotang % 180.0, 0.0):
563 width = r[0]
564 height = r[1]
565 else:
566 width = r[1]
567 height = r[0]
568 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
569 lsst.geom.Extent2I(width, height))
570 else:
571 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
572 continue
573 else:
574 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
575 continue
577 else:
578 # This is a classic LSST-style defect table
579 box = lsst.geom.Box2I(lsst.geom.Point2I(record['x0'], record['y0']),
580 lsst.geom.Extent2I(record['width'], record['height']))
582 defectList.append(box)
584 defects = cls(defectList, normalize_on_init=normalize_on_init)
585 newMeta = dict(table.meta)
586 defects.updateMetadata(setCalibInfo=True, **newMeta)
588 return defects
590 @classmethod
591 def readLsstDefectsFile(cls, filename, normalize_on_init=False):
592 """Read defects information from a legacy LSST format text file.
594 Parameters
595 ----------
596 filename : `str`
597 Name of text file containing the defect information.
599 normalize_on_init : `bool`, optional
600 If `True`, normalization is applied to the defects listed in the
601 table to remove duplicates, eliminate overlaps, etc. Otherwise
602 the defects in the returned object exactly match those in the
603 table.
605 Returns
606 -------
607 defects : `Defects`
608 The defects.
610 Notes
611 -----
612 These defect text files are used as the human readable definitions
613 of defects in calibration data definition repositories. The format
614 is to use four columns defined as follows:
616 x0 : `int`
617 X coordinate of bottom left corner of box.
618 y0 : `int`
619 Y coordinate of bottom left corner of box.
620 width : `int`
621 X extent of the box.
622 height : `int`
623 Y extent of the box.
625 Files of this format were used historically to represent defects
626 in simple text form. Use `Defects.readText` and `Defects.writeText`
627 to use the more modern format.
628 """
629 # Use loadtxt so that ValueError is thrown if the file contains a
630 # non-integer value. genfromtxt converts bad values to -1.
631 defect_array = np.loadtxt(filename,
632 dtype=[("x0", "int"), ("y0", "int"),
633 ("x_extent", "int"), ("y_extent", "int")])
635 defects = (lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
636 lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
637 for row in defect_array)
639 return cls(defects, normalize_on_init=normalize_on_init)
641 @classmethod
642 def fromFootprintList(cls, fpList):
643 """Compute a defect list from a footprint list, optionally growing
644 the footprints.
646 Parameters
647 ----------
648 fpList : `list` of `lsst.afw.detection.Footprint`
649 Footprint list to process.
651 Returns
652 -------
653 defects : `Defects`
654 List of defects.
655 """
656 # normalize_on_init is set to False to avoid recursively calling
657 # fromMask/fromFootprintList in Defects.__init__.
658 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
659 for fp in fpList), normalize_on_init=False)
661 @classmethod
662 def fromMask(cls, maskedImage, maskName):
663 """Compute a defect list from a specified mask plane.
665 Parameters
666 ----------
667 maskedImage : `lsst.afw.image.MaskedImage`
668 Image to process.
669 maskName : `str` or `list`
670 Mask plane name, or list of names to convert.
672 Returns
673 -------
674 defects : `Defects`
675 Defect list constructed from masked pixels.
676 """
677 mask = maskedImage.getMask()
678 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
679 lsst.afw.detection.Threshold.BITMASK)
680 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
681 return cls.fromFootprintList(fpList)