lsst.meas.algorithms  19.0.0-9-gce87a591+2
defects.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Support for image defects"""
24 
25 __all__ = ("Defects",)
26 
27 import logging
28 import itertools
29 import collections.abc
30 import numpy as np
31 import copy
32 import datetime
33 import math
34 import numbers
35 import os.path
36 import astropy.table
37 
38 import lsst.geom
39 import lsst.afw.table
40 import lsst.afw.detection
41 import lsst.afw.image
42 import lsst.afw.geom
43 from lsst.daf.base import PropertyList
44 
45 from . import Defect
46 
47 log = logging.getLogger(__name__)
48 
49 SCHEMA_NAME_KEY = "DEFECTS_SCHEMA"
50 SCHEMA_VERSION_KEY = "DEFECTS_SCHEMA_VERSION"
51 
52 
53 class Defects(collections.abc.MutableSequence):
54  """Collection of `lsst.meas.algorithms.Defect`.
55 
56  Parameters
57  ----------
58  defectList : iterable of `lsst.meas.algorithms.Defect`
59  or `lsst.geom.BoxI`, optional
60  Collections of defects to apply to the image.
61  """
62 
63  _OBSTYPE = "defects"
64  """The calibration type used for ingest."""
65 
66  def __init__(self, defectList=None, metadata=None):
67  self._defects = []
68 
69  if metadata is not None:
70  self._metadata = metadata
71  else:
72  self.setMetadata()
73 
74  if defectList is None:
75  return
76 
77  # Ensure that type checking
78  for d in defectList:
79  self.append(d)
80 
81  def _check_value(self, value):
82  """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
83  or can be converted to one.
84 
85  Parameters
86  ----------
87  value : `object`
88  Value to check.
89 
90  Returns
91  -------
92  new : `~lsst.meas.algorithms.Defect`
93  Either the supplied value or a new object derived from it.
94 
95  Raises
96  ------
97  ValueError
98  Raised if the supplied value can not be converted to
99  `~lsst.meas.algorithms.Defect`
100  """
101  if isinstance(value, Defect):
102  pass
103  elif isinstance(value, lsst.geom.BoxI):
104  value = Defect(value)
105  elif isinstance(value, lsst.geom.PointI):
106  value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
107  elif isinstance(value, lsst.afw.image.DefectBase):
108  value = Defect(value.getBBox())
109  else:
110  raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
111  return value
112 
113  def __len__(self):
114  return len(self._defects)
115 
116  def __getitem__(self, index):
117  return self._defects[index]
118 
119  def __setitem__(self, index, value):
120  """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
121  """
122  self._defects[index] = self._check_value(value)
123 
124  def __iter__(self):
125  return iter(self._defects)
126 
127  def __delitem__(self, index):
128  del self._defects[index]
129 
130  def __eq__(self, other):
131  """Compare if two `Defects` are equal.
132 
133  Two `Defects` are equal if their bounding boxes are equal and in
134  the same order. Metadata content is ignored.
135  """
136  if not isinstance(other, self.__class__):
137  return False
138 
139  # checking the bboxes with zip() only works if same length
140  if len(self) != len(other):
141  return False
142 
143  # Assume equal if bounding boxes are equal
144  for d1, d2 in zip(self, other):
145  if d1.getBBox() != d2.getBBox():
146  return False
147 
148  return True
149 
150  def __str__(self):
151  return "Defects(" + ",".join(str(d.getBBox()) for d in self) + ")"
152 
153  def insert(self, index, value):
154  self._defects.insert(index, self._check_value(value))
155 
156  def getMetadata(self):
157  """Retrieve metadata associated with these `Defects`.
158 
159  Returns
160  -------
161  meta : `lsst.daf.base.PropertyList`
162  Metadata. The returned `~lsst.daf.base.PropertyList` can be
163  modified by the caller and the changes will be written to
164  external files.
165  """
166  return self._metadata
167 
168  def setMetadata(self, metadata=None):
169  """Store a copy of the supplied metadata with the defects.
170 
171  Parameters
172  ----------
173  metadata : `lsst.daf.base.PropertyList`, optional
174  Metadata to associate with the defects. Will be copied and
175  overwrite existing metadata. If not supplied the existing
176  metadata will be reset.
177  """
178  if metadata is None:
179  self._metadata = PropertyList()
180  else:
181  self._metadata = copy.copy(metadata)
182 
183  # Ensure that we have the obs type required by calibration ingest
184  self._metadata["OBSTYPE"] = self._OBSTYPE
185 
186  def copy(self):
187  """Copy the defects to a new list, creating new defects from the
188  bounding boxes.
189 
190  Returns
191  -------
192  new : `Defects`
193  New list with new `Defect` entries.
194 
195  Notes
196  -----
197  This is not a shallow copy in that new `Defect` instances are
198  created from the original bounding boxes. It's also not a deep
199  copy since the bounding boxes are not recreated.
200  """
201  return self.__class__(d.getBBox() for d in self)
202 
203  def transpose(self):
204  """Make a transposed copy of this defect list.
205 
206  Returns
207  -------
208  retDefectList : `Defects`
209  Transposed list of defects.
210  """
211  retDefectList = self.__class__()
212  for defect in self:
213  bbox = defect.getBBox()
214  dimensions = bbox.getDimensions()
215  nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
216  lsst.geom.Extent2I(dimensions[1], dimensions[0]))
217  retDefectList.append(nbbox)
218  return retDefectList
219 
220  def maskPixels(self, maskedImage, maskName="BAD"):
221  """Set mask plane based on these defects.
222 
223  Parameters
224  ----------
225  maskedImage : `lsst.afw.image.MaskedImage`
226  Image to process. Only the mask plane is updated.
227  maskName : str, optional
228  Mask plane name to use.
229  """
230  # mask bad pixels
231  mask = maskedImage.getMask()
232  bitmask = mask.getPlaneBitMask(maskName)
233  for defect in self:
234  bbox = defect.getBBox()
235  lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
236 
237  def toFitsRegionTable(self):
238  """Convert defect list to `~lsst.afw.table.BaseCatalog` using the
239  FITS region standard.
240 
241  Returns
242  -------
243  table : `lsst.afw.table.BaseCatalog`
244  Defects in tabular form.
245 
246  Notes
247  -----
248  The table created uses the
249  `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
250  definition tabular format. The ``X`` and ``Y`` coordinates are
251  converted to FITS Physical coordinates that have origin pixel (1, 1)
252  rather than the (0, 0) used in LSST software.
253  """
254 
255  nrows = len(self._defects)
256 
257  schema = lsst.afw.table.Schema()
258  x = schema.addField("X", type="D", units="pix", doc="X coordinate of center of shape")
259  y = schema.addField("Y", type="D", units="pix", doc="Y coordinate of center of shape")
260  shape = schema.addField("SHAPE", type="String", size=16, doc="Shape defined by these values")
261  r = schema.addField("R", type="ArrayD", size=2, units="pix", doc="Extents")
262  rotang = schema.addField("ROTANG", type="D", units="deg", doc="Rotation angle")
263  component = schema.addField("COMPONENT", type="I", doc="Index of this region")
264  table = lsst.afw.table.BaseCatalog(schema)
265  table.resize(nrows)
266 
267  if nrows:
268  # Adding entire columns is more efficient than adding
269  # each element separately
270  xCol = []
271  yCol = []
272  rCol = []
273 
274  for i, defect in enumerate(self._defects):
275  box = defect.getBBox()
276  center = box.getCenter()
277  # Correct for the FITS 1-based offset
278  xCol.append(center.getX() + 1.0)
279  yCol.append(center.getY() + 1.0)
280 
281  width = box.width
282  height = box.height
283 
284  if width == 1 and height == 1:
285  # Call this a point
286  shapeType = "POINT"
287  else:
288  shapeType = "BOX"
289 
290  # Strings have to be added per row
291  table[i][shape] = shapeType
292 
293  rCol.append(np.array([width, height], dtype=np.float64))
294 
295  # Assign the columns
296  table[x] = np.array(xCol, dtype=np.float64)
297  table[y] = np.array(yCol, dtype=np.float64)
298 
299  table[r] = np.array(rCol)
300  table[rotang] = np.zeros(nrows, dtype=np.float64)
301  table[component] = np.arange(nrows)
302 
303  # Set some metadata in the table (force OBSTYPE to exist)
304  metadata = copy.copy(self.getMetadata())
305  metadata["OBSTYPE"] = self._OBSTYPE
306  metadata[SCHEMA_NAME_KEY] = "FITS Region"
307  metadata[SCHEMA_VERSION_KEY] = 1
308  table.setMetadata(metadata)
309 
310  return table
311 
312  def writeFits(self, *args):
313  """Write defect list to FITS.
314 
315  Parameters
316  ----------
317  *args
318  Arguments to be forwarded to
319  `lsst.afw.table.BaseCatalog.writeFits`.
320  """
321  table = self.toFitsRegionTable()
322 
323  # Add some additional headers useful for tracking purposes
324  metadata = table.getMetadata()
325  now = datetime.datetime.utcnow()
326  metadata["DATE"] = now.isoformat()
327  metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
328  metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
329 
330  table.writeFits(*args)
331 
332  def toSimpleTable(self):
333  """Convert defects to a simple table form that we use to write
334  to text files.
335 
336  Returns
337  -------
338  table : `lsst.afw.table.BaseCatalog`
339  Defects in simple tabular form.
340 
341  Notes
342  -----
343  These defect tables are used as the human readable definitions
344  of defects in calibration data definition repositories. The format
345  is to use four columns defined as follows:
346 
347  x0 : `int`
348  X coordinate of bottom left corner of box.
349  y0 : `int`
350  Y coordinate of bottom left corner of box.
351  width : `int`
352  X extent of the box.
353  height : `int`
354  Y extent of the box.
355  """
356  schema = lsst.afw.table.Schema()
357  x = schema.addField("x0", type="I", units="pix",
358  doc="X coordinate of bottom left corner of box")
359  y = schema.addField("y0", type="I", units="pix",
360  doc="Y coordinate of bottom left corner of box")
361  width = schema.addField("width", type="I", units="pix",
362  doc="X extent of box")
363  height = schema.addField("height", type="I", units="pix",
364  doc="Y extent of box")
365  table = lsst.afw.table.BaseCatalog(schema)
366 
367  nrows = len(self._defects)
368  table.resize(nrows)
369 
370  if nrows:
371 
372  xCol = []
373  yCol = []
374  widthCol = []
375  heightCol = []
376 
377  for defect in self._defects:
378  box = defect.getBBox()
379  xCol.append(box.getBeginX())
380  yCol.append(box.getBeginY())
381  widthCol.append(box.getWidth())
382  heightCol.append(box.getHeight())
383 
384  table[x] = np.array(xCol, dtype=np.int64)
385  table[y] = np.array(yCol, dtype=np.int64)
386  table[width] = np.array(widthCol, dtype=np.int64)
387  table[height] = np.array(heightCol, dtype=np.int64)
388 
389  # Set some metadata in the table (force OBSTYPE to exist)
390  metadata = copy.copy(self.getMetadata())
391  metadata["OBSTYPE"] = self._OBSTYPE
392  metadata[SCHEMA_NAME_KEY] = "Simple"
393  metadata[SCHEMA_VERSION_KEY] = 1
394  table.setMetadata(metadata)
395 
396  return table
397 
398  def writeText(self, filename):
399  """Write the defects out to a text file with the specified name.
400 
401  Parameters
402  ----------
403  filename : `str`
404  Name of the file to write. The file extension ".ecsv" will
405  always be used.
406 
407  Returns
408  -------
409  used : `str`
410  The name of the file used to write the data (which may be
411  different from the supplied name given the change to file
412  extension).
413 
414  Notes
415  -----
416  The file is written to ECSV format and will include any metadata
417  associated with the `Defects`.
418  """
419 
420  # Using astropy table is the easiest way to serialize to ecsv
421  afwTable = self.toSimpleTable()
422  table = afwTable.asAstropy()
423 
424  metadata = afwTable.getMetadata()
425  now = datetime.datetime.utcnow()
426  metadata["DATE"] = now.isoformat()
427  metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
428  metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
429 
430  table.meta = metadata.toDict()
431 
432  # Force file extension to .ecsv
433  path, ext = os.path.splitext(filename)
434  filename = path + ".ecsv"
435  table.write(filename, format="ascii.ecsv")
436  return filename
437 
438  @staticmethod
439  def _get_values(values, n=1):
440  """Retrieve N values from the supplied values.
441 
442  Parameters
443  ----------
444  values : `numbers.Number` or `list` or `np.array`
445  Input values.
446  n : `int`
447  Number of values to retrieve.
448 
449  Returns
450  -------
451  vals : `list` or `np.array` or `numbers.Number`
452  Single value from supplied list if ``n`` is 1, or `list`
453  containing first ``n`` values from supplied values.
454 
455  Notes
456  -----
457  Some supplied tables have vectors in some columns that can also
458  be scalars. This method can be used to get the first number as
459  a scalar or the first N items from a vector as a vector.
460  """
461  if n == 1:
462  if isinstance(values, numbers.Number):
463  return values
464  else:
465  return values[0]
466 
467  return values[:n]
468 
469  @classmethod
470  def fromTable(cls, table):
471  """Construct a `Defects` from the contents of a
472  `~lsst.afw.table.BaseCatalog`.
473 
474  Parameters
475  ----------
476  table : `lsst.afw.table.BaseCatalog`
477  Table with one row per defect.
478 
479  Returns
480  -------
481  defects : `Defects`
482  A `Defects` list.
483 
484  Notes
485  -----
486  Two table formats are recognized. The first is the
487  `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
488  definition tabular format written by `toFitsRegionTable` where the
489  pixel origin is corrected from FITS 1-based to a 0-based origin.
490  The second is the legacy defects format using columns ``x0``, ``y0``
491  (bottom left hand pixel of box in 0-based coordinates), ``width``
492  and ``height``.
493 
494  The FITS standard regions can only read BOX, POINT, or ROTBOX with
495  a zero degree rotation.
496  """
497 
498  defectList = []
499 
500  schema = table.getSchema()
501 
502  # Check schema to see which definitions we have
503  if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
504  # This is a FITS region style table
505  isFitsRegion = True
506 
507  # Preselect the keys
508  xKey = schema["X"].asKey()
509  yKey = schema["Y"].asKey()
510  shapeKey = schema["SHAPE"].asKey()
511  rKey = schema["R"].asKey()
512  rotangKey = schema["ROTANG"].asKey()
513 
514  elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
515  # This is a classic LSST-style defect table
516  isFitsRegion = False
517 
518  # Preselect the keys
519  xKey = schema["x0"].asKey()
520  yKey = schema["y0"].asKey()
521  widthKey = schema["width"].asKey()
522  heightKey = schema["height"].asKey()
523 
524  else:
525  raise ValueError("Unsupported schema for defects extraction")
526 
527  for record in table:
528 
529  if isFitsRegion:
530  # Coordinates can be arrays (some shapes in the standard
531  # require this)
532  # Correct for FITS 1-based origin
533  xcen = cls._get_values(record[xKey]) - 1.0
534  ycen = cls._get_values(record[yKey]) - 1.0
535  shape = record[shapeKey].upper()
536  if shape == "BOX":
538  lsst.geom.Extent2I(cls._get_values(record[rKey],
539  n=2)))
540  elif shape == "POINT":
541  # Handle the case where we have an externally created
542  # FITS file.
543  box = lsst.geom.Point2I(xcen, ycen)
544  elif shape == "ROTBOX":
545  # Astropy regions always writes ROTBOX
546  rotang = cls._get_values(record[rotangKey])
547  # We can support 0 or 90 deg
548  if math.isclose(rotang % 90.0, 0.0):
549  # Two values required
550  r = cls._get_values(record[rKey], n=2)
551  if math.isclose(rotang % 180.0, 0.0):
552  width = r[0]
553  height = r[1]
554  else:
555  width = r[1]
556  height = r[0]
558  lsst.geom.Extent2I(width, height))
559  else:
560  log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
561  continue
562  else:
563  log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
564  continue
565 
566  else:
567  # This is a classic LSST-style defect table
568  box = lsst.geom.Box2I(lsst.geom.Point2I(record[xKey], record[yKey]),
569  lsst.geom.Extent2I(record[widthKey], record[heightKey]))
570 
571  defectList.append(box)
572 
573  defects = cls(defectList)
574  defects.setMetadata(table.getMetadata())
575 
576  # Once read, the schema headers are irrelevant
577  metadata = defects.getMetadata()
578  for k in (SCHEMA_NAME_KEY, SCHEMA_VERSION_KEY):
579  if k in metadata:
580  del metadata[k]
581 
582  return defects
583 
584  @classmethod
585  def readFits(cls, *args):
586  """Read defect list from FITS table.
587 
588  Parameters
589  ----------
590  *args
591  Arguments to be forwarded to
592  `lsst.afw.table.BaseCatalog.writeFits`.
593 
594  Returns
595  -------
596  defects : `Defects`
597  Defects read from a FITS table.
598  """
599  table = lsst.afw.table.BaseCatalog.readFits(*args)
600  return cls.fromTable(table)
601 
602  @classmethod
603  def readText(cls, filename):
604  """Read defect list from standard format text table file.
605 
606  Parameters
607  ----------
608  filename : `str`
609  Name of the file containing the defects definitions.
610 
611  Returns
612  -------
613  defects : `Defects`
614  Defects read from a FITS table.
615  """
616  table = astropy.table.Table.read(filename)
617 
618  # Need to convert the Astropy table to afw table
619  schema = lsst.afw.table.Schema()
620  for colName in table.columns:
621  schema.addField(colName, units=str(table[colName].unit),
622  type=table[colName].dtype.type)
623 
624  # Create AFW table that is required by fromTable()
625  afwTable = lsst.afw.table.BaseCatalog(schema)
626 
627  afwTable.resize(len(table))
628  for colName in table.columns:
629  # String columns will fail -- currently we do not expect any
630  afwTable[colName] = table[colName]
631 
632  # Copy in the metadata from the astropy table
633  metadata = PropertyList()
634  for k, v in table.meta.items():
635  metadata[k] = v
636  afwTable.setMetadata(metadata)
637 
638  # Extract defect information from the table itself
639  return cls.fromTable(afwTable)
640 
641  @classmethod
642  def readLsstDefectsFile(cls, filename):
643  """Read defects information from a legacy LSST format text file.
644 
645  Parameters
646  ----------
647  filename : `str`
648  Name of text file containing the defect information.
649 
650  Returns
651  -------
652  defects : `Defects`
653  The defects.
654 
655  Notes
656  -----
657  These defect text files are used as the human readable definitions
658  of defects in calibration data definition repositories. The format
659  is to use four columns defined as follows:
660 
661  x0 : `int`
662  X coordinate of bottom left corner of box.
663  y0 : `int`
664  Y coordinate of bottom left corner of box.
665  width : `int`
666  X extent of the box.
667  height : `int`
668  Y extent of the box.
669 
670  Files of this format were used historically to represent defects
671  in simple text form. Use `Defects.readText` and `Defects.writeText`
672  to use the more modern format.
673  """
674  # Use loadtxt so that ValueError is thrown if the file contains a
675  # non-integer value. genfromtxt converts bad values to -1.
676  defect_array = np.loadtxt(filename,
677  dtype=[("x0", "int"), ("y0", "int"),
678  ("x_extent", "int"), ("y_extent", "int")])
679 
680  return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
681  lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
682  for row in defect_array)
683 
684  @classmethod
685  def fromFootprintList(cls, fpList):
686  """Compute a defect list from a footprint list, optionally growing
687  the footprints.
688 
689  Parameters
690  ----------
691  fpList : `list` of `lsst.afw.detection.Footprint`
692  Footprint list to process.
693 
694  Returns
695  -------
696  defects : `Defects`
697  List of defects.
698  """
699  return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
700  for fp in fpList))
701 
702  @classmethod
703  def fromMask(cls, maskedImage, maskName):
704  """Compute a defect list from a specified mask plane.
705 
706  Parameters
707  ----------
708  maskedImage : `lsst.afw.image.MaskedImage`
709  Image to process.
710  maskName : `str` or `list`
711  Mask plane name, or list of names to convert.
712 
713  Returns
714  -------
715  defects : `Defects`
716  Defect list constructed from masked pixels.
717  """
718  mask = maskedImage.getMask()
719  thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
720  lsst.afw.detection.Threshold.BITMASK)
721  fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
722  return cls.fromFootprintList(fpList)
def maskPixels(self, maskedImage, maskName="BAD")
Definition: defects.py:220
def setMetadata(self, metadata=None)
Definition: defects.py:168
def __setitem__(self, index, value)
Definition: defects.py:119
def __init__(self, defectList=None, metadata=None)
Definition: defects.py:66
def readLsstDefectsFile(cls, filename)
Definition: defects.py:642
std::vector< lsst::geom::Box2I > footprintToBBoxList(Footprint const &footprint)
def insert(self, index, value)
Definition: defects.py:153
static Box2I makeCenteredBox(Point2D const &center, Extent const &size)
def fromMask(cls, maskedImage, maskName)
Definition: defects.py:703