lsst.meas.algorithms  17.0.1-11-gf0f4e679+8
defects.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Support for image defects"""
24 
25 __all__ = ("Defects",)
26 
27 import logging
28 import itertools
29 import collections.abc
30 from deprecated.sphinx import deprecated
31 import numpy as np
32 import copy
33 import datetime
34 import math
35 import numbers
36 
37 import lsst.geom
38 import lsst.pex.policy as policy
39 import lsst.afw.table
40 import lsst.afw.detection
41 import lsst.afw.image
42 import lsst.afw.geom
43 from lsst.daf.base import PropertyList
44 
45 from . import Defect
46 
47 log = logging.getLogger(__name__)
48 
49 
50 @deprecated(reason="Policy defect files no longer supported (will be removed after v18)",
51  category=FutureWarning)
52 def policyToBadRegionList(policyFile):
53  """Given a Policy file describing a CCD's bad pixels, return a vector of BadRegion::Ptr"""
54 
55  badPixelsPolicy = policy.Policy.createPolicy(policyFile)
56  badPixels = []
57 
58  if badPixelsPolicy.exists("Defects"):
59  d = badPixelsPolicy.getArray("Defects")
60  for reg in d:
61  x0 = reg.get("x0")
62  width = reg.get("width")
63  if not width:
64  x1 = reg.get("x1")
65  width = x1 - x0 - 1
66 
67  y0 = reg.get("y0")
68  if reg.exists("height"):
69  height = reg.get("height")
70  else:
71  y1 = reg.get("y1")
72  height = y1 - y0 - 1
73 
74  bbox = lsst.geom.BoxI(lsst.geom.PointI(x0, y0), lsst.geom.ExtentI(width, height))
75  badPixels.append(Defect(bbox))
76 
77  return badPixels
78 
79 
80 class Defects(collections.abc.MutableSequence):
81  """Collection of `lsst.meas.algorithms.Defect`.
82 
83  Parameters
84  ----------
85  defectList : iterable of `lsst.meas.algorithms.Defect`
86  or `lsst.geom.BoxI`, optional
87  Collections of defects to apply to the image.
88  """
89 
90  _OBSTYPE = "defects"
91  """The calibration type used for ingest."""
92 
93  def __init__(self, defectList=None, metadata=None):
94  self._defects = []
95 
96  if metadata is not None:
97  self._metadata = metadata
98  else:
99  self.setMetadata()
100 
101  if defectList is None:
102  return
103 
104  # Ensure that type checking
105  for d in defectList:
106  self.append(d)
107 
108  def _check_value(self, value):
109  """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
110  or can be converted to one.
111 
112  Parameters
113  ----------
114  value : `object`
115  Value to check.
116 
117  Returns
118  -------
119  new : `~lsst.meas.algorithms.Defect`
120  Either the supplied value or a new object derived from it.
121 
122  Raises
123  ------
124  ValueError
125  Raised if the supplied value can not be converted to
126  `~lsst.meas.algorithms.Defect`
127  """
128  if isinstance(value, Defect):
129  pass
130  elif isinstance(value, lsst.geom.BoxI):
131  value = Defect(value)
132  elif isinstance(value, lsst.geom.PointI):
133  value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
134  elif isinstance(value, lsst.afw.image.DefectBase):
135  value = Defect(value.getBBox())
136  else:
137  raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
138  return value
139 
140  def __len__(self):
141  return len(self._defects)
142 
143  def __getitem__(self, index):
144  return self._defects[index]
145 
146  def __setitem__(self, index, value):
147  """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
148  """
149  self._defects[index] = self._check_value(value)
150 
151  def __iter__(self):
152  return iter(self._defects)
153 
154  def __delitem__(self, index):
155  del self._defects[index]
156 
157  def __eq__(self, other):
158  """Compare if two `Defects` are equal.
159 
160  Two `Defects` are equal if their bounding boxes are equal and in
161  the same order. Metadata content is ignored.
162  """
163  if not isinstance(other, self.__class__):
164  return False
165 
166  # Assume equal if bounding boxes are equal
167  for d1, d2 in zip(self, other):
168  if d1.getBBox() != d2.getBBox():
169  return False
170 
171  return True
172 
173  def __str__(self):
174  return "Defects(" + ",".join(str(d.getBBox()) for d in self) + ")"
175 
176  def insert(self, index, value):
177  self._defects.insert(index, self._check_value(value))
178 
179  def getMetadata(self):
180  """Retrieve metadata associated with these `Defects`.
181 
182  Returns
183  -------
184  meta : `lsst.daf.base.PropertyList`
185  Metadata. The returned `~lsst.daf.base.PropertyList` can be
186  modified by the caller and the changes will be written to
187  external files.
188  """
189  return self._metadata
190 
191  def setMetadata(self, metadata=None):
192  """Store a copy of the supplied metadata with the defects.
193 
194  Parameters
195  ----------
196  metadata : `lsst.daf.base.PropertyList`, optional
197  Metadata to associate with the defects. Will be copied and
198  overwrite existing metadata. If not supplied the existing
199  metadata will be reset.
200  """
201  if metadata is None:
202  self._metadata = PropertyList()
203  else:
204  self._metadata = copy.copy(metadata)
205 
206  # Ensure that we have the obs type required by calibration ingest
207  self._metadata["OBSTYPE"] = self._OBSTYPE
208 
209  def copy(self):
210  """Copy the defects to a new list, creating new defects from the
211  bounding boxes.
212 
213  Returns
214  -------
215  new : `Defects`
216  New list with new `Defect` entries.
217 
218  Notes
219  -----
220  This is not a shallow copy in that new `Defect` instances are
221  created from the original bounding boxes. It's also not a deep
222  copy since the bounding boxes are not recreated.
223  """
224  return self.__class__(d.getBBox() for d in self)
225 
226  def transpose(self):
227  """Make a transposed copy of this defect list.
228 
229  Returns
230  -------
231  retDefectList : `Defects`
232  Transposed list of defects.
233  """
234  retDefectList = self.__class__()
235  for defect in self:
236  bbox = defect.getBBox()
237  dimensions = bbox.getDimensions()
238  nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
239  lsst.geom.Extent2I(dimensions[1], dimensions[0]))
240  retDefectList.append(nbbox)
241  return retDefectList
242 
243  def maskPixels(self, maskedImage, maskName="BAD"):
244  """Set mask plane based on these defects.
245 
246  Parameters
247  ----------
248  maskedImage : `lsst.afw.image.MaskedImage`
249  Image to process. Only the mask plane is updated.
250  maskName : str, optional
251  Mask plane name to use.
252  """
253  # mask bad pixels
254  mask = maskedImage.getMask()
255  bitmask = mask.getPlaneBitMask(maskName)
256  for defect in self:
257  bbox = defect.getBBox()
258  lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
259 
260  def toTable(self):
261  """Convert defect list to `~lsst.afw.table.BaseCatalog`
262 
263  Returns
264  -------
265  table : `lsst.afw.table.BaseCatalog`
266  Defects in tabular form.
267 
268  Notes
269  -----
270  The table created uses the
271  `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
272  definition tabular format. The ``X`` and ``Y`` coordinates are
273  converted to FITS Physical coordinates that have origin pixel (1, 1)
274  rather than the (0, 0) used in LSST software.
275  """
276  schema = lsst.afw.table.Schema()
277  x = schema.addField("X", type="D", units="pix")
278  y = schema.addField("Y", type="D", units="pix")
279  shape = schema.addField("SHAPE", type="String", size=16)
280  r = schema.addField("R", type="ArrayD", size=2, units="pix")
281  rotang = schema.addField("ROTANG", type="D", units="deg")
282  component = schema.addField("COMPONENT", type="I")
283  table = lsst.afw.table.BaseCatalog(schema)
284 
285  for i, defect in enumerate(self._defects):
286  box = defect.getBBox()
287  record = table.addNew()
288  # Correct for the FITS 1-based offset
289  record.set(x, box.getCenterX() + 1.0)
290  record.set(y, box.getCenterY() + 1.0)
291  width = box.getWidth()
292  height = box.getHeight()
293 
294  if width == 1 and height == 1:
295  # Call this a point
296  shapeType = "POINT"
297  else:
298  shapeType = "BOX"
299  record.set(shape, shapeType)
300  record.set(r, np.array([width, height], dtype=np.float64))
301  record.set(rotang, 0.0)
302  record.set(component, i)
303 
304  # Set some metadata in the table (force OBSTYPE to exist)
305  metadata = copy.copy(self.getMetadata())
306  metadata["OBSTYPE"] = self._OBSTYPE
307  table.setMetadata(metadata)
308 
309  return table
310 
311  def writeFits(self, *args):
312  """Write defect list to FITS.
313 
314  Parameters
315  ----------
316  *args
317  Arguments to be forwarded to
318  `lsst.afw.table.BaseCatalog.writeFits`.
319  """
320  table = self.toTable()
321 
322  # Add some additional headers useful for tracking purposes
323  metadata = table.getMetadata()
324  now = datetime.datetime.utcnow()
325  metadata["DATE"] = now.isoformat()
326  metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
327  metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
328 
329  table.writeFits(*args)
330 
331  @staticmethod
332  def _get_values(values, n=1):
333  """Retrieve N values from the supplied values.
334 
335  Parameters
336  ----------
337  values : `numbers.Number` or `list` or `np.array`
338  Input values.
339  n : `int`
340  Number of values to retrieve.
341 
342  Returns
343  -------
344  vals : `list` or `np.array` or `numbers.Number`
345  Single value from supplied list if ``n`` is 1, or `list`
346  containing first ``n`` values from supplied values.
347 
348  Notes
349  -----
350  Some supplied tables have vectors in some columns that can also
351  be scalars. This method can be used to get the first number as
352  a scalar or the first N items from a vector as a vector.
353  """
354  if n == 1:
355  if isinstance(values, numbers.Number):
356  return values
357  else:
358  return values[0]
359 
360  return values[:n]
361 
362  @classmethod
363  def fromTable(cls, table):
364  """Construct a `Defects` from the contents of a
365  `~lsst.afw.table.BaseCatalog`.
366 
367  Parameters
368  ----------
369  table : `lsst.afw.table.BaseCatalog`
370  Table with one row per defect.
371 
372  Returns
373  -------
374  defects : `Defects`
375  A `Defects` list.
376 
377  Notes
378  -----
379  Two table formats are recognized. The first is the
380  `FITS regions <https://fits.gsfc.nasa.gov/registry/region.html>`_
381  definition tabular format written by `toTable` where the pixel origin
382  is corrected from FITS 1-based to a 0-based origin. The second is
383  the legacy defects format using columns ``x0``, ``y0`` (bottom left
384  hand pixel of box in 0-based coordinates), ``width`` and ``height``.
385 
386  The FITS standard regions can only read BOX, POINT, or ROTBOX with
387  a zero degree rotation.
388  """
389 
390  defectList = []
391 
392  schema = table.getSchema()
393 
394  # Check schema to see which definitions we have
395  if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
396  # This is a FITS region style table
397  isFitsRegion = True
398 
399  elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
400  # This is a classic LSST-style defect table
401  isFitsRegion = False
402 
403  else:
404  raise ValueError("Unsupported schema for defects extraction")
405 
406  for r in table:
407  record = r.extract("*")
408 
409  if isFitsRegion:
410  # Coordinates can be arrays (some shapes in the standard
411  # require this)
412  # Correct for FITS 1-based origin
413  xcen = cls._get_values(record["X"]) - 1.0
414  ycen = cls._get_values(record["Y"]) - 1.0
415  shape = record["SHAPE"].upper()
416  if shape == "BOX":
418  lsst.geom.Extent2I(cls._get_values(record["R"],
419  n=2)))
420  elif shape == "POINT":
421  # Handle the case where we have an externally created
422  # FITS file.
423  box = lsst.geom.Point2I(xcen, ycen)
424  elif shape == "ROTBOX":
425  # Astropy regions always writes ROTBOX
426  rotang = cls._get_values(record["ROTANG"])
427  # We can support 0 or 90 deg
428  if math.isclose(rotang % 90.0, 0.0):
429  # Two values required
430  r = cls._get_values(record["R"], n=2)
431  if math.isclose(rotang % 180.0, 0.0):
432  width = r[0]
433  height = r[1]
434  else:
435  width = r[1]
436  height = r[0]
438  lsst.geom.Extent2I(width, height))
439  else:
440  log.warning("Defect can not be defined using ROTBOX with non-aligned rotation angle")
441  continue
442  else:
443  log.warning("Defect lists can only be defined using BOX or POINT not %s", shape)
444  continue
445 
446  elif "x0" in record and "y0" in record and "width" in record and "height" in record:
447  # This is a classic LSST-style defect table
448  box = lsst.geom.Box2I(lsst.geom.Point2I(record["x0"], record["y0"]),
449  lsst.geom.Extent2I(record["width"], record["height"]))
450 
451  defectList.append(box)
452 
453  return cls(defectList)
454 
455  @classmethod
456  def readFits(cls, *args):
457  """Read defect list from FITS table.
458 
459  Parameters
460  ----------
461  *args
462  Arguments to be forwarded to
463  `lsst.afw.table.BaseCatalog.writeFits`.
464 
465  Returns
466  -------
467  defects : `Defects`
468  Defects read from a FITS table.
469  """
470  table = lsst.afw.table.BaseCatalog.readFits(*args)
471  defects = cls.fromTable(table)
472  defects.setMetadata(table.getMetadata())
473  return defects
474 
475  @classmethod
476  def readLsstDefectsFile(cls, filename):
477  """Read defects information from an LSST format text file.
478 
479  Parameters
480  ----------
481  filename : `str`
482  Name of text file containing the defect information.
483 
484  Returns
485  -------
486  defects : `Defects`
487  The defects.
488 
489  Notes
490  -----
491  These defect text files are used as the human readable definitions
492  of defects in calibration data definition repositories. The format
493  is to use four columns defined as follows:
494 
495  x0 : `int`
496  X coordinate of bottom left corner of box.
497  y0 : `int`
498  Y coordinate of bottom left corner of box.
499  width : `int`
500  X extent of the box.
501  height : `int`
502  Y extent of the box.
503  """
504  # Use loadtxt so that ValueError is thrown if the file contains a
505  # non-integer value. genfromtxt converts bad values to -1.
506  defect_array = np.loadtxt(filename,
507  dtype=[("x0", "int"), ("y0", "int"),
508  ("x_extent", "int"), ("y_extent", "int")])
509 
510  return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
511  lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
512  for row in defect_array)
513 
514  @classmethod
515  def fromFootprintList(cls, fpList):
516  """Compute a defect list from a footprint list, optionally growing
517  the footprints.
518 
519  Parameters
520  ----------
521  fpList : `list` of `lsst.afw.detection.Footprint`
522  Footprint list to process.
523 
524  Returns
525  -------
526  defects : `Defects`
527  List of defects.
528  """
529  return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
530  for fp in fpList))
531 
532  @classmethod
533  def fromMask(cls, maskedImage, maskName):
534  """Compute a defect list from a specified mask plane.
535 
536  Parameters
537  ----------
538  maskedImage : `lsst.afw.image.MaskedImage`
539  Image to process.
540  maskName : `str` or `list`
541  Mask plane name, or list of names to convert.
542 
543  Returns
544  -------
545  defects : `Defects`
546  Defect list constructed from masked pixels.
547  """
548  mask = maskedImage.getMask()
549  thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
550  lsst.afw.detection.Threshold.BITMASK)
551  fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
552  return cls.fromFootprintList(fpList)
def maskPixels(self, maskedImage, maskName="BAD")
Definition: defects.py:243
def setMetadata(self, metadata=None)
Definition: defects.py:191
def __setitem__(self, index, value)
Definition: defects.py:146
def __init__(self, defectList=None, metadata=None)
Definition: defects.py:93
def policyToBadRegionList(policyFile)
Definition: defects.py:52
def readLsstDefectsFile(cls, filename)
Definition: defects.py:476
std::vector< lsst::geom::Box2I > footprintToBBoxList(Footprint const &footprint)
def insert(self, index, value)
Definition: defects.py:176
static Box2I makeCenteredBox(Point2D const &center, Extent const &size)
def fromMask(cls, maskedImage, maskName)
Definition: defects.py:533