Coverage for python/lsst/meas/algorithms/defects.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23"""Support for image defects"""
25__all__ = ("Defects",)
27import logging
28import itertools
29import collections.abc
30import numpy as np
31import copy
32import datetime
33import math
34import numbers
35import os.path
36import astropy.table
38import lsst.geom
39import lsst.afw.table
40import lsst.afw.detection
41import lsst.afw.image
42import lsst.afw.geom
43from lsst.daf.base import PropertyList
45from . import Defect
47log = logging.getLogger(__name__)
49SCHEMA_NAME_KEY = "DEFECTS_SCHEMA"
50SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION"
53class Defects(collections.abc.MutableSequence):
54 """Collection of `lsst.meas.algorithms.Defect`.
56 Parameters
57 ----------
58 defectList : iterable of `lsst.meas.algorithms.Defect`
59 or `lsst.geom.BoxI`, optional
60 Collections of defects to apply to the image.
61 """
63 _OBSTYPE = "defects"
64 """The calibration type used for ingest."""
66 def __init__(self, defectList=None, metadata=None):
67 self._defects = []
69 if metadata is not None:
70 self._metadata = metadata
71 else:
72 self.setMetadata()
74 if defectList is None:
75 return
77 # Ensure that type checking
78 for d in defectList:
79 self.append(d)
81 def _check_value(self, value):
82 """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
83 or can be converted to one.
85 Parameters
86 ----------
87 value : `object`
88 Value to check.
90 Returns
91 -------
92 new : `~lsst.meas.algorithms.Defect`
93 Either the supplied value or a new object derived from it.
95 Raises
96 ------
97 ValueError
98 Raised if the supplied value can not be converted to
99 `~lsst.meas.algorithms.Defect`
100 """
101 if isinstance(value, Defect):
102 pass
103 elif isinstance(value, lsst.geom.BoxI):
104 value = Defect(value)
105 elif isinstance(value, lsst.geom.PointI):
106 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
107 elif isinstance(value, lsst.afw.image.DefectBase):
108 value = Defect(value.getBBox())
109 else:
110 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
111 return value
113 def __len__(self):
114 return len(self._defects)
116 def __getitem__(self, index):
117 return self._defects[index]
119 def __setitem__(self, index, value):
120 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
121 """
122 self._defects[index] = self._check_value(value)
124 def __iter__(self):
125 return iter(self._defects)
127 def __delitem__(self, index):
128 del self._defects[index]
130 def __eq__(self, other):
131 """Compare if two `Defects` are equal.
133 Two `Defects` are equal if their bounding boxes are equal and in
134 the same order. Metadata content is ignored.
135 """
136 if not isinstance(other, self.__class__):
137 return False
139 # checking the bboxes with zip() only works if same length
140 if len(self) != len(other):
141 return False
143 # Assume equal if bounding boxes are equal
144 for d1, d2 in zip(self, other):
145 if d1.getBBox() != d2.getBBox():
146 return False
148 return True
150 def __str__(self):
151 return "Defects(" + ",".join(str(d.getBBox()) for d in self) + ")"
153 def insert(self, index, value):
154 self._defects.insert(index, self._check_value(value))
156 def getMetadata(self):
157 """Retrieve metadata associated with these `Defects`.
159 Returns
160 -------
161 meta : `lsst.daf.base.PropertyList`
162 Metadata. The returned `~lsst.daf.base.PropertyList` can be
163 modified by the caller and the changes will be written to
164 external files.
165 """
166 return self._metadata
168 def setMetadata(self, metadata=None):
169 """Store a copy of the supplied metadata with the defects.
171 Parameters
172 ----------
173 metadata : `lsst.daf.base.PropertyList`, optional
174 Metadata to associate with the defects. Will be copied and
175 overwrite existing metadata. If not supplied the existing
176 metadata will be reset.
177 """
178 if metadata is None:
179 self._metadata = PropertyList()
180 else:
181 self._metadata = copy.copy(metadata)
183 # Ensure that we have the obs type required by calibration ingest
184 self._metadata["OBSTYPE"] = self._OBSTYPE
186 def copy(self):
187 """Copy the defects to a new list, creating new defects from the
188 bounding boxes.
190 Returns
191 -------
192 new : `Defects`
193 New list with new `Defect` entries.
195 Notes
196 -----
197 This is not a shallow copy in that new `Defect` instances are
198 created from the original bounding boxes. It's also not a deep
199 copy since the bounding boxes are not recreated.
200 """
201 return self.__class__(d.getBBox() for d in self)
203 def transpose(self):
204 """Make a transposed copy of this defect list.
206 Returns
207 -------
208 retDefectList : `Defects`
209 Transposed list of defects.
210 """
211 retDefectList = self.__class__()
212 for defect in self:
213 bbox = defect.getBBox()
214 dimensions = bbox.getDimensions()
215 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
216 lsst.geom.Extent2I(dimensions[1], dimensions[0]))
217 retDefectList.append(nbbox)
218 return retDefectList
220 def maskPixels(self, maskedImage, maskName="BAD"):
221 """Set mask plane based on these defects.
223 Parameters
224 ----------
225 maskedImage : `lsst.afw.image.MaskedImage`
226 Image to process. Only the mask plane is updated.
227 maskName : str, optional
228 Mask plane name to use.
229 """
230 # mask bad pixels
231 mask = maskedImage.getMask()
232 bitmask = mask.getPlaneBitMask(maskName)
233 for defect in self:
234 bbox = defect.getBBox()
235 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
237 def toFitsRegionTable(self):
238 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the
239 FITS region standard.
241 Returns
242 -------
243 table : `lsst.afw.table.BaseCatalog`
244 Defects in tabular form.
246 Notes
247 -----
248 The table created uses the
249 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
250 definition tabular format. The ``X`` and ``Y`` coordinates are
251 converted to FITS Physical coordinates that have origin pixel (1, 1)
252 rather than the (0, 0) used in LSST software.
253 """
255 nrows = len(self._defects)
257 schema = lsst.afw.table.Schema()
258 x = schema.addField("X", type="D", units="pix", doc="X coordinate of center of shape")
259 y = schema.addField("Y", type="D", units="pix", doc="Y coordinate of center of shape")
260 shape = schema.addField("SHAPE", type="String", size=16, doc="Shape defined by these values")
261 r = schema.addField("R", type="ArrayD", size=2, units="pix", doc="Extents")
262 rotang = schema.addField("ROTANG", type="D", units="deg", doc="Rotation angle")
263 component = schema.addField("COMPONENT", type="I", doc="Index of this region")
264 table = lsst.afw.table.BaseCatalog(schema)
265 table.resize(nrows)
267 if nrows:
268 # Adding entire columns is more efficient than adding
269 # each element separately
270 xCol = []
271 yCol = []
272 rCol = []
274 for i, defect in enumerate(self._defects):
275 box = defect.getBBox()
276 center = box.getCenter()
277 # Correct for the FITS 1-based offset
278 xCol.append(center.getX() + 1.0)
279 yCol.append(center.getY() + 1.0)
281 width = box.width
282 height = box.height
284 if width == 1 and height == 1:
285 # Call this a point
286 shapeType = "POINT"
287 else:
288 shapeType = "BOX"
290 # Strings have to be added per row
291 table[i][shape] = shapeType
293 rCol.append(np.array([width, height], dtype=np.float64))
295 # Assign the columns
296 table[x] = np.array(xCol, dtype=np.float64)
297 table[y] = np.array(yCol, dtype=np.float64)
299 table[r] = np.array(rCol)
300 table[rotang] = np.zeros(nrows, dtype=np.float64)
301 table[component] = np.arange(nrows)
303 # Set some metadata in the table (force OBSTYPE to exist)
304 metadata = copy.copy(self.getMetadata())
305 metadata["OBSTYPE"] = self._OBSTYPE
306 metadata[SCHEMA_NAME_KEY] = "FITS Region"
307 metadata[SCHEMA_VERSION_KEY] = 1
308 table.setMetadata(metadata)
310 return table
312 def writeFits(self, *args):
313 """Write defect list to FITS.
315 Parameters
316 ----------
317 *args
318 Arguments to be forwarded to
319 `lsst.afw.table.BaseCatalog.writeFits`.
320 """
321 table = self.toFitsRegionTable()
323 # Add some additional headers useful for tracking purposes
324 metadata = table.getMetadata()
325 now = datetime.datetime.utcnow()
326 metadata["DATE"] = now.isoformat()
327 metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
328 metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
330 table.writeFits(*args)
332 def toSimpleTable(self):
333 """Convert defects to a simple table form that we use to write
334 to text files.
336 Returns
337 -------
338 table : `lsst.afw.table.BaseCatalog`
339 Defects in simple tabular form.
341 Notes
342 -----
343 These defect tables are used as the human readable definitions
344 of defects in calibration data definition repositories. The format
345 is to use four columns defined as follows:
347 x0 : `int`
348 X coordinate of bottom left corner of box.
349 y0 : `int`
350 Y coordinate of bottom left corner of box.
351 width : `int`
352 X extent of the box.
353 height : `int`
354 Y extent of the box.
355 """
356 schema = lsst.afw.table.Schema()
357 x = schema.addField("x0", type="I", units="pix",
358 doc="X coordinate of bottom left corner of box")
359 y = schema.addField("y0", type="I", units="pix",
360 doc="Y coordinate of bottom left corner of box")
361 width = schema.addField("width", type="I", units="pix",
362 doc="X extent of box")
363 height = schema.addField("height", type="I", units="pix",
364 doc="Y extent of box")
365 table = lsst.afw.table.BaseCatalog(schema)
367 nrows = len(self._defects)
368 table.resize(nrows)
370 if nrows:
372 xCol = []
373 yCol = []
374 widthCol = []
375 heightCol = []
377 for defect in self._defects:
378 box = defect.getBBox()
379 xCol.append(box.getBeginX())
380 yCol.append(box.getBeginY())
381 widthCol.append(box.getWidth())
382 heightCol.append(box.getHeight())
384 table[x] = np.array(xCol, dtype=np.int64)
385 table[y] = np.array(yCol, dtype=np.int64)
386 table[width] = np.array(widthCol, dtype=np.int64)
387 table[height] = np.array(heightCol, dtype=np.int64)
389 # Set some metadata in the table (force OBSTYPE to exist)
390 metadata = copy.copy(self.getMetadata())
391 metadata["OBSTYPE"] = self._OBSTYPE
392 metadata[SCHEMA_NAME_KEY] = "Simple"
393 metadata[SCHEMA_VERSION_KEY] = 1
394 table.setMetadata(metadata)
396 return table
398 def writeText(self, filename):
399 """Write the defects out to a text file with the specified name.
401 Parameters
402 ----------
403 filename : `str`
404 Name of the file to write. The file extension ".ecsv" will
405 always be used.
407 Returns
408 -------
409 used : `str`
410 The name of the file used to write the data (which may be
411 different from the supplied name given the change to file
412 extension).
414 Notes
415 -----
416 The file is written to ECSV format and will include any metadata
417 associated with the `Defects`.
418 """
420 # Using astropy table is the easiest way to serialize to ecsv
421 afwTable = self.toSimpleTable()
422 table = afwTable.asAstropy()
424 metadata = afwTable.getMetadata()
425 now = datetime.datetime.utcnow()
426 metadata["DATE"] = now.isoformat()
427 metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
428 metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
430 table.meta = metadata.toDict()
432 # Force file extension to .ecsv
433 path, ext = os.path.splitext(filename)
434 filename = path + ".ecsv"
435 table.write(filename, format="ascii.ecsv")
436 return filename
438 @staticmethod
439 def _get_values(values, n=1):
440 """Retrieve N values from the supplied values.
442 Parameters
443 ----------
444 values : `numbers.Number` or `list` or `np.array`
445 Input values.
446 n : `int`
447 Number of values to retrieve.
449 Returns
450 -------
451 vals : `list` or `np.array` or `numbers.Number`
452 Single value from supplied list if ``n`` is 1, or `list`
453 containing first ``n`` values from supplied values.
455 Notes
456 -----
457 Some supplied tables have vectors in some columns that can also
458 be scalars. This method can be used to get the first number as
459 a scalar or the first N items from a vector as a vector.
460 """
461 if n == 1:
462 if isinstance(values, numbers.Number):
463 return values
464 else:
465 return values[0]
467 return values[:n]
469 @classmethod
470 def fromTable(cls, table):
471 """Construct a `Defects` from the contents of a
472 `~lsst.afw.table.BaseCatalog`.
474 Parameters
475 ----------
476 table : `lsst.afw.table.BaseCatalog`
477 Table with one row per defect.
479 Returns
480 -------
481 defects : `Defects`
482 A `Defects` list.
484 Notes
485 -----
486 Two table formats are recognized. The first is the
487 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
488 definition tabular format written by `toFitsRegionTable` where the
489 pixel origin is corrected from FITS 1-based to a 0-based origin.
490 The second is the legacy defects format using columns ``x0``, ``y0``
491 (bottom left hand pixel of box in 0-based coordinates), ``width``
492 and ``height``.
494 The FITS standard regions can only read BOX, POINT, or ROTBOX with
495 a zero degree rotation.
496 """
498 defectList = []
500 schema = table.getSchema()
502 # Check schema to see which definitions we have
503 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
504 # This is a FITS region style table
505 isFitsRegion = True
507 # Preselect the keys
508 xKey = schema["X"].asKey()
509 yKey = schema["Y"].asKey()
510 shapeKey = schema["SHAPE"].asKey()
511 rKey = schema["R"].asKey()
512 rotangKey = schema["ROTANG"].asKey()
514 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
515 # This is a classic LSST-style defect table
516 isFitsRegion = False
518 # Preselect the keys
519 xKey = schema["x0"].asKey()
520 yKey = schema["y0"].asKey()
521 widthKey = schema["width"].asKey()
522 heightKey = schema["height"].asKey()
524 else:
525 raise ValueError("Unsupported schema for defects extraction")
527 for record in table:
529 if isFitsRegion:
530 # Coordinates can be arrays (some shapes in the standard
531 # require this)
532 # Correct for FITS 1-based origin
533 xcen = cls._get_values(record[xKey]) - 1.0
534 ycen = cls._get_values(record[yKey]) - 1.0
535 shape = record[shapeKey].upper()
536 if shape == "BOX":
537 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
538 lsst.geom.Extent2I(cls._get_values(record[rKey],
539 n=2)))
540 elif shape == "POINT":
541 # Handle the case where we have an externally created
542 # FITS file.
543 box = lsst.geom.Point2I(xcen, ycen)
544 elif shape == "ROTBOX":
545 # Astropy regions always writes ROTBOX
546 rotang = cls._get_values(record[rotangKey])
547 # We can support 0 or 90 deg
548 if math.isclose(rotang % 90.0, 0.0):
549 # Two values required
550 r = cls._get_values(record[rKey], n=2)
551 if math.isclose(rotang % 180.0, 0.0):
552 width = r[0]
553 height = r[1]
554 else:
555 width = r[1]
556 height = r[0]
557 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
558 lsst.geom.Extent2I(width, height))
559 else:
560 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
561 continue
562 else:
563 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
564 continue
566 else:
567 # This is a classic LSST-style defect table
568 box = lsst.geom.Box2I(lsst.geom.Point2I(record[xKey], record[yKey]),
569 lsst.geom.Extent2I(record[widthKey], record[heightKey]))
571 defectList.append(box)
573 defects = cls(defectList)
574 defects.setMetadata(table.getMetadata())
576 # Once read, the schema headers are irrelevant
577 metadata = defects.getMetadata()
578 for k in (SCHEMA_NAME_KEY, SCHEMA_VERSION_KEY):
579 if k in metadata:
580 del metadata[k]
582 return defects
584 @classmethod
585 def readFits(cls, *args):
586 """Read defect list from FITS table.
588 Parameters
589 ----------
590 *args
591 Arguments to be forwarded to
592 `lsst.afw.table.BaseCatalog.writeFits`.
594 Returns
595 -------
596 defects : `Defects`
597 Defects read from a FITS table.
598 """
599 table = lsst.afw.table.BaseCatalog.readFits(*args)
600 return cls.fromTable(table)
602 @classmethod
603 def readText(cls, filename):
604 """Read defect list from standard format text table file.
606 Parameters
607 ----------
608 filename : `str`
609 Name of the file containing the defects definitions.
611 Returns
612 -------
613 defects : `Defects`
614 Defects read from a FITS table.
615 """
616 table = astropy.table.Table.read(filename)
618 # Need to convert the Astropy table to afw table
619 schema = lsst.afw.table.Schema()
620 for colName in table.columns:
621 schema.addField(colName, units=str(table[colName].unit),
622 type=table[colName].dtype.type)
624 # Create AFW table that is required by fromTable()
625 afwTable = lsst.afw.table.BaseCatalog(schema)
627 afwTable.resize(len(table))
628 for colName in table.columns:
629 # String columns will fail -- currently we do not expect any
630 afwTable[colName] = table[colName]
632 # Copy in the metadata from the astropy table
633 metadata = PropertyList()
634 for k, v in table.meta.items():
635 metadata[k] = v
636 afwTable.setMetadata(metadata)
638 # Extract defect information from the table itself
639 return cls.fromTable(afwTable)
641 @classmethod
642 def readLsstDefectsFile(cls, filename):
643 """Read defects information from a legacy LSST format text file.
645 Parameters
646 ----------
647 filename : `str`
648 Name of text file containing the defect information.
650 Returns
651 -------
652 defects : `Defects`
653 The defects.
655 Notes
656 -----
657 These defect text files are used as the human readable definitions
658 of defects in calibration data definition repositories. The format
659 is to use four columns defined as follows:
661 x0 : `int`
662 X coordinate of bottom left corner of box.
663 y0 : `int`
664 Y coordinate of bottom left corner of box.
665 width : `int`
666 X extent of the box.
667 height : `int`
668 Y extent of the box.
670 Files of this format were used historically to represent defects
671 in simple text form. Use `Defects.readText` and `Defects.writeText`
672 to use the more modern format.
673 """
674 # Use loadtxt so that ValueError is thrown if the file contains a
675 # non-integer value. genfromtxt converts bad values to -1.
676 defect_array = np.loadtxt(filename,
677 dtype=[("x0", "int"), ("y0", "int"),
678 ("x_extent", "int"), ("y_extent", "int")])
680 return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
681 lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
682 for row in defect_array)
684 @classmethod
685 def fromFootprintList(cls, fpList):
686 """Compute a defect list from a footprint list, optionally growing
687 the footprints.
689 Parameters
690 ----------
691 fpList : `list` of `lsst.afw.detection.Footprint`
692 Footprint list to process.
694 Returns
695 -------
696 defects : `Defects`
697 List of defects.
698 """
699 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
700 for fp in fpList))
702 @classmethod
703 def fromMask(cls, maskedImage, maskName):
704 """Compute a defect list from a specified mask plane.
706 Parameters
707 ----------
708 maskedImage : `lsst.afw.image.MaskedImage`
709 Image to process.
710 maskName : `str` or `list`
711 Mask plane name, or list of names to convert.
713 Returns
714 -------
715 defects : `Defects`
716 Defect list constructed from masked pixels.
717 """
718 mask = maskedImage.getMask()
719 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
720 lsst.afw.detection.Threshold.BITMASK)
721 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
722 return cls.fromFootprintList(fpList)