Coverage for python/lsst/meas/algorithms/defects.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23"""Support for image defects"""
25__all__ = ("Defects",)
27import logging
28import itertools
29import collections.abc
30import numpy as np
31import copy
32import datetime
33import math
34import numbers
35import os.path
36import warnings
37import astropy.table
39import lsst.geom
40import lsst.afw.table
41import lsst.afw.detection
42import lsst.afw.image
43import lsst.afw.geom
44from lsst.daf.base import PropertyList
46from . import Defect
48log = logging.getLogger(__name__)
50SCHEMA_NAME_KEY = "DEFECTS_SCHEMA"
51SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION"
54class Defects(collections.abc.MutableSequence):
55 """Collection of `lsst.meas.algorithms.Defect`.
57 Parameters
58 ----------
59 defectList : iterable of `lsst.meas.algorithms.Defect`
60 or `lsst.geom.BoxI`, optional
61 Collections of defects to apply to the image.
62 """
64 _OBSTYPE = "defects"
65 """The calibration type used for ingest."""
67 def __init__(self, defectList=None, metadata=None):
68 self._defects = []
70 if metadata is not None:
71 self._metadata = metadata
72 else:
73 self.setMetadata()
75 if defectList is None:
76 return
78 # Ensure that type checking
79 for d in defectList:
80 self.append(d)
82 def _check_value(self, value):
83 """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
84 or can be converted to one.
86 Parameters
87 ----------
88 value : `object`
89 Value to check.
91 Returns
92 -------
93 new : `~lsst.meas.algorithms.Defect`
94 Either the supplied value or a new object derived from it.
96 Raises
97 ------
98 ValueError
99 Raised if the supplied value can not be converted to
100 `~lsst.meas.algorithms.Defect`
101 """
102 if isinstance(value, Defect):
103 pass
104 elif isinstance(value, lsst.geom.BoxI):
105 value = Defect(value)
106 elif isinstance(value, lsst.geom.PointI):
107 value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
108 elif isinstance(value, lsst.afw.image.DefectBase):
109 value = Defect(value.getBBox())
110 else:
111 raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
112 return value
114 def __len__(self):
115 return len(self._defects)
117 def __getitem__(self, index):
118 return self._defects[index]
120 def __setitem__(self, index, value):
121 """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
122 """
123 self._defects[index] = self._check_value(value)
125 def __iter__(self):
126 return iter(self._defects)
128 def __delitem__(self, index):
129 del self._defects[index]
131 def __eq__(self, other):
132 """Compare if two `Defects` are equal.
134 Two `Defects` are equal if their bounding boxes are equal and in
135 the same order. Metadata content is ignored.
136 """
137 if not isinstance(other, self.__class__):
138 return False
140 # checking the bboxes with zip() only works if same length
141 if len(self) != len(other):
142 return False
144 # Assume equal if bounding boxes are equal
145 for d1, d2 in zip(self, other):
146 if d1.getBBox() != d2.getBBox():
147 return False
149 return True
151 def __str__(self):
152 return "Defects(" + ",".join(str(d.getBBox()) for d in self) + ")"
154 def insert(self, index, value):
155 self._defects.insert(index, self._check_value(value))
157 def getMetadata(self):
158 """Retrieve metadata associated with these `Defects`.
160 Returns
161 -------
162 meta : `lsst.daf.base.PropertyList`
163 Metadata. The returned `~lsst.daf.base.PropertyList` can be
164 modified by the caller and the changes will be written to
165 external files.
166 """
167 return self._metadata
169 def setMetadata(self, metadata=None):
170 """Store a copy of the supplied metadata with the defects.
172 Parameters
173 ----------
174 metadata : `lsst.daf.base.PropertyList`, optional
175 Metadata to associate with the defects. Will be copied and
176 overwrite existing metadata. If not supplied the existing
177 metadata will be reset.
178 """
179 if metadata is None:
180 self._metadata = PropertyList()
181 else:
182 self._metadata = copy.copy(metadata)
184 # Ensure that we have the obs type required by calibration ingest
185 self._metadata["OBSTYPE"] = self._OBSTYPE
187 def copy(self):
188 """Copy the defects to a new list, creating new defects from the
189 bounding boxes.
191 Returns
192 -------
193 new : `Defects`
194 New list with new `Defect` entries.
196 Notes
197 -----
198 This is not a shallow copy in that new `Defect` instances are
199 created from the original bounding boxes. It's also not a deep
200 copy since the bounding boxes are not recreated.
201 """
202 return self.__class__(d.getBBox() for d in self)
204 def transpose(self):
205 """Make a transposed copy of this defect list.
207 Returns
208 -------
209 retDefectList : `Defects`
210 Transposed list of defects.
211 """
212 retDefectList = self.__class__()
213 for defect in self:
214 bbox = defect.getBBox()
215 dimensions = bbox.getDimensions()
216 nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
217 lsst.geom.Extent2I(dimensions[1], dimensions[0]))
218 retDefectList.append(nbbox)
219 return retDefectList
221 def maskPixels(self, maskedImage, maskName="BAD"):
222 """Set mask plane based on these defects.
224 Parameters
225 ----------
226 maskedImage : `lsst.afw.image.MaskedImage`
227 Image to process. Only the mask plane is updated.
228 maskName : str, optional
229 Mask plane name to use.
230 """
231 # mask bad pixels
232 mask = maskedImage.getMask()
233 bitmask = mask.getPlaneBitMask(maskName)
234 for defect in self:
235 bbox = defect.getBBox()
236 lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
238 def toFitsRegionTable(self):
239 """Convert defect list to `~lsst.afw.table.BaseCatalog` using the
240 FITS region standard.
242 Returns
243 -------
244 table : `lsst.afw.table.BaseCatalog`
245 Defects in tabular form.
247 Notes
248 -----
249 The table created uses the
250 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
251 definition tabular format. The ``X`` and ``Y`` coordinates are
252 converted to FITS Physical coordinates that have origin pixel (1, 1)
253 rather than the (0, 0) used in LSST software.
254 """
256 nrows = len(self._defects)
258 schema = lsst.afw.table.Schema()
259 x = schema.addField("X", type="D", units="pix", doc="X coordinate of center of shape")
260 y = schema.addField("Y", type="D", units="pix", doc="Y coordinate of center of shape")
261 shape = schema.addField("SHAPE", type="String", size=16, doc="Shape defined by these values")
262 r = schema.addField("R", type="ArrayD", size=2, units="pix", doc="Extents")
263 rotang = schema.addField("ROTANG", type="D", units="deg", doc="Rotation angle")
264 component = schema.addField("COMPONENT", type="I", doc="Index of this region")
265 table = lsst.afw.table.BaseCatalog(schema)
266 table.resize(nrows)
268 if nrows:
269 # Adding entire columns is more efficient than adding
270 # each element separately
271 xCol = []
272 yCol = []
273 rCol = []
275 for i, defect in enumerate(self._defects):
276 box = defect.getBBox()
277 center = box.getCenter()
278 # Correct for the FITS 1-based offset
279 xCol.append(center.getX() + 1.0)
280 yCol.append(center.getY() + 1.0)
282 width = box.width
283 height = box.height
285 if width == 1 and height == 1:
286 # Call this a point
287 shapeType = "POINT"
288 else:
289 shapeType = "BOX"
291 # Strings have to be added per row
292 table[i][shape] = shapeType
294 rCol.append(np.array([width, height], dtype=np.float64))
296 # Assign the columns
297 table[x] = np.array(xCol, dtype=np.float64)
298 table[y] = np.array(yCol, dtype=np.float64)
300 table[r] = np.array(rCol)
301 table[rotang] = np.zeros(nrows, dtype=np.float64)
302 table[component] = np.arange(nrows)
304 # Set some metadata in the table (force OBSTYPE to exist)
305 metadata = copy.copy(self.getMetadata())
306 metadata["OBSTYPE"] = self._OBSTYPE
307 metadata[SCHEMA_NAME_KEY] = "FITS Region"
308 metadata[SCHEMA_VERSION_KEY] = 1
309 table.setMetadata(metadata)
311 return table
313 def writeFits(self, *args):
314 """Write defect list to FITS.
316 Parameters
317 ----------
318 *args
319 Arguments to be forwarded to
320 `lsst.afw.table.BaseCatalog.writeFits`.
321 """
322 table = self.toFitsRegionTable()
324 # Add some additional headers useful for tracking purposes
325 metadata = table.getMetadata()
326 now = datetime.datetime.utcnow()
327 metadata["DATE"] = now.isoformat()
328 metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
329 metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
331 table.writeFits(*args)
333 def toSimpleTable(self):
334 """Convert defects to a simple table form that we use to write
335 to text files.
337 Returns
338 -------
339 table : `lsst.afw.table.BaseCatalog`
340 Defects in simple tabular form.
342 Notes
343 -----
344 These defect tables are used as the human readable definitions
345 of defects in calibration data definition repositories. The format
346 is to use four columns defined as follows:
348 x0 : `int`
349 X coordinate of bottom left corner of box.
350 y0 : `int`
351 Y coordinate of bottom left corner of box.
352 width : `int`
353 X extent of the box.
354 height : `int`
355 Y extent of the box.
356 """
357 schema = lsst.afw.table.Schema()
358 x = schema.addField("x0", type="I", units="pix",
359 doc="X coordinate of bottom left corner of box")
360 y = schema.addField("y0", type="I", units="pix",
361 doc="Y coordinate of bottom left corner of box")
362 width = schema.addField("width", type="I", units="pix",
363 doc="X extent of box")
364 height = schema.addField("height", type="I", units="pix",
365 doc="Y extent of box")
366 table = lsst.afw.table.BaseCatalog(schema)
368 nrows = len(self._defects)
369 table.resize(nrows)
371 if nrows:
373 xCol = []
374 yCol = []
375 widthCol = []
376 heightCol = []
378 for defect in self._defects:
379 box = defect.getBBox()
380 xCol.append(box.getBeginX())
381 yCol.append(box.getBeginY())
382 widthCol.append(box.getWidth())
383 heightCol.append(box.getHeight())
385 table[x] = np.array(xCol, dtype=np.int64)
386 table[y] = np.array(yCol, dtype=np.int64)
387 table[width] = np.array(widthCol, dtype=np.int64)
388 table[height] = np.array(heightCol, dtype=np.int64)
390 # Set some metadata in the table (force OBSTYPE to exist)
391 metadata = copy.copy(self.getMetadata())
392 metadata["OBSTYPE"] = self._OBSTYPE
393 metadata[SCHEMA_NAME_KEY] = "Simple"
394 metadata[SCHEMA_VERSION_KEY] = 1
395 table.setMetadata(metadata)
397 return table
399 def writeText(self, filename):
400 """Write the defects out to a text file with the specified name.
402 Parameters
403 ----------
404 filename : `str`
405 Name of the file to write. The file extension ".ecsv" will
406 always be used.
408 Returns
409 -------
410 used : `str`
411 The name of the file used to write the data (which may be
412 different from the supplied name given the change to file
413 extension).
415 Notes
416 -----
417 The file is written to ECSV format and will include any metadata
418 associated with the `Defects`.
419 """
421 # Using astropy table is the easiest way to serialize to ecsv
422 afwTable = self.toSimpleTable()
423 table = afwTable.asAstropy()
425 metadata = afwTable.getMetadata()
426 now = datetime.datetime.utcnow()
427 metadata["DATE"] = now.isoformat()
428 metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
429 metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
431 table.meta = metadata.toDict()
433 # Force file extension to .ecsv
434 path, ext = os.path.splitext(filename)
435 filename = path + ".ecsv"
436 table.write(filename, format="ascii.ecsv")
437 return filename
439 @staticmethod
440 def _get_values(values, n=1):
441 """Retrieve N values from the supplied values.
443 Parameters
444 ----------
445 values : `numbers.Number` or `list` or `np.array`
446 Input values.
447 n : `int`
448 Number of values to retrieve.
450 Returns
451 -------
452 vals : `list` or `np.array` or `numbers.Number`
453 Single value from supplied list if ``n`` is 1, or `list`
454 containing first ``n`` values from supplied values.
456 Notes
457 -----
458 Some supplied tables have vectors in some columns that can also
459 be scalars. This method can be used to get the first number as
460 a scalar or the first N items from a vector as a vector.
461 """
462 if n == 1:
463 if isinstance(values, numbers.Number):
464 return values
465 else:
466 return values[0]
468 return values[:n]
470 @classmethod
471 def fromTable(cls, table):
472 """Construct a `Defects` from the contents of a
473 `~lsst.afw.table.BaseCatalog`.
475 Parameters
476 ----------
477 table : `lsst.afw.table.BaseCatalog`
478 Table with one row per defect.
480 Returns
481 -------
482 defects : `Defects`
483 A `Defects` list.
485 Notes
486 -----
487 Two table formats are recognized. The first is the
488 `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
489 definition tabular format written by `toFitsRegionTable` where the
490 pixel origin is corrected from FITS 1-based to a 0-based origin.
491 The second is the legacy defects format using columns ``x0``, ``y0``
492 (bottom left hand pixel of box in 0-based coordinates), ``width``
493 and ``height``.
495 The FITS standard regions can only read BOX, POINT, or ROTBOX with
496 a zero degree rotation.
497 """
499 defectList = []
501 schema = table.getSchema()
503 # Check schema to see which definitions we have
504 if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
505 # This is a FITS region style table
506 isFitsRegion = True
508 # Preselect the keys
509 xKey = schema["X"].asKey()
510 yKey = schema["Y"].asKey()
511 shapeKey = schema["SHAPE"].asKey()
512 rKey = schema["R"].asKey()
513 rotangKey = schema["ROTANG"].asKey()
515 elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
516 # This is a classic LSST-style defect table
517 isFitsRegion = False
519 # Preselect the keys
520 xKey = schema["x0"].asKey()
521 yKey = schema["y0"].asKey()
522 widthKey = schema["width"].asKey()
523 heightKey = schema["height"].asKey()
525 else:
526 raise ValueError("Unsupported schema for defects extraction")
528 for record in table:
530 if isFitsRegion:
531 # Coordinates can be arrays (some shapes in the standard
532 # require this)
533 # Correct for FITS 1-based origin
534 xcen = cls._get_values(record[xKey]) - 1.0
535 ycen = cls._get_values(record[yKey]) - 1.0
536 shape = record[shapeKey].upper()
537 if shape == "BOX":
538 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
539 lsst.geom.Extent2I(cls._get_values(record[rKey],
540 n=2)))
541 elif shape == "POINT":
542 # Handle the case where we have an externally created
543 # FITS file.
544 box = lsst.geom.Point2I(xcen, ycen)
545 elif shape == "ROTBOX":
546 # Astropy regions always writes ROTBOX
547 rotang = cls._get_values(record[rotangKey])
548 # We can support 0 or 90 deg
549 if math.isclose(rotang % 90.0, 0.0):
550 # Two values required
551 r = cls._get_values(record[rKey], n=2)
552 if math.isclose(rotang % 180.0, 0.0):
553 width = r[0]
554 height = r[1]
555 else:
556 width = r[1]
557 height = r[0]
558 box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(xcen, ycen),
559 lsst.geom.Extent2I(width, height))
560 else:
561 log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
562 continue
563 else:
564 log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
565 continue
567 else:
568 # This is a classic LSST-style defect table
569 box = lsst.geom.Box2I(lsst.geom.Point2I(record[xKey], record[yKey]),
570 lsst.geom.Extent2I(record[widthKey], record[heightKey]))
572 defectList.append(box)
574 defects = cls(defectList)
575 defects.setMetadata(table.getMetadata())
577 # Once read, the schema headers are irrelevant
578 metadata = defects.getMetadata()
579 for k in (SCHEMA_NAME_KEY, SCHEMA_VERSION_KEY):
580 if k in metadata:
581 del metadata[k]
583 return defects
585 @classmethod
586 def readFits(cls, *args):
587 """Read defect list from FITS table.
589 Parameters
590 ----------
591 *args
592 Arguments to be forwarded to
593 `lsst.afw.table.BaseCatalog.writeFits`.
595 Returns
596 -------
597 defects : `Defects`
598 Defects read from a FITS table.
599 """
600 table = lsst.afw.table.BaseCatalog.readFits(*args)
601 return cls.fromTable(table)
603 @classmethod
604 def readText(cls, filename):
605 """Read defect list from standard format text table file.
607 Parameters
608 ----------
609 filename : `str`
610 Name of the file containing the defects definitions.
612 Returns
613 -------
614 defects : `Defects`
615 Defects read from a FITS table.
616 """
617 with warnings.catch_warnings():
618 # Squash warnings due to astropy failure to close files; we think
619 # this is a real problem, but the warnings are even worse.
620 # https://github.com/astropy/astropy/issues/8673
621 warnings.filterwarnings("ignore", category=ResourceWarning, module="astropy.io.ascii")
622 table = astropy.table.Table.read(filename)
624 # Need to convert the Astropy table to afw table
625 schema = lsst.afw.table.Schema()
626 for colName in table.columns:
627 schema.addField(colName, units=str(table[colName].unit),
628 type=table[colName].dtype.type)
630 # Create AFW table that is required by fromTable()
631 afwTable = lsst.afw.table.BaseCatalog(schema)
633 afwTable.resize(len(table))
634 for colName in table.columns:
635 # String columns will fail -- currently we do not expect any
636 afwTable[colName] = table[colName]
638 # Copy in the metadata from the astropy table
639 metadata = PropertyList()
640 for k, v in table.meta.items():
641 metadata[k] = v
642 afwTable.setMetadata(metadata)
644 # Extract defect information from the table itself
645 return cls.fromTable(afwTable)
647 @classmethod
648 def readLsstDefectsFile(cls, filename):
649 """Read defects information from a legacy LSST format text file.
651 Parameters
652 ----------
653 filename : `str`
654 Name of text file containing the defect information.
656 Returns
657 -------
658 defects : `Defects`
659 The defects.
661 Notes
662 -----
663 These defect text files are used as the human readable definitions
664 of defects in calibration data definition repositories. The format
665 is to use four columns defined as follows:
667 x0 : `int`
668 X coordinate of bottom left corner of box.
669 y0 : `int`
670 Y coordinate of bottom left corner of box.
671 width : `int`
672 X extent of the box.
673 height : `int`
674 Y extent of the box.
676 Files of this format were used historically to represent defects
677 in simple text form. Use `Defects.readText` and `Defects.writeText`
678 to use the more modern format.
679 """
680 # Use loadtxt so that ValueError is thrown if the file contains a
681 # non-integer value. genfromtxt converts bad values to -1.
682 defect_array = np.loadtxt(filename,
683 dtype=[("x0", "int"), ("y0", "int"),
684 ("x_extent", "int"), ("y_extent", "int")])
686 return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
687 lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
688 for row in defect_array)
690 @classmethod
691 def fromFootprintList(cls, fpList):
692 """Compute a defect list from a footprint list, optionally growing
693 the footprints.
695 Parameters
696 ----------
697 fpList : `list` of `lsst.afw.detection.Footprint`
698 Footprint list to process.
700 Returns
701 -------
702 defects : `Defects`
703 List of defects.
704 """
705 return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
706 for fp in fpList))
708 @classmethod
709 def fromMask(cls, maskedImage, maskName):
710 """Compute a defect list from a specified mask plane.
712 Parameters
713 ----------
714 maskedImage : `lsst.afw.image.MaskedImage`
715 Image to process.
716 maskName : `str` or `list`
717 Mask plane name, or list of names to convert.
719 Returns
720 -------
721 defects : `Defects`
722 Defect list constructed from masked pixels.
723 """
724 mask = maskedImage.getMask()
725 thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
726 lsst.afw.detection.Threshold.BITMASK)
727 fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
728 return cls.fromFootprintList(fpList)