lsst.meas.algorithms  17.0.1-10-g6c37d570
defects.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Support for image defects"""
24 
25 __all__ = ("Defects",)
26 
27 import logging
28 import itertools
29 import collections.abc
30 from deprecated.sphinx import deprecated
31 import numpy as np
32 import copy
33 import datetime
34 
35 import lsst.geom
36 import lsst.pex.policy as policy
37 import lsst.afw.table
38 import lsst.afw.detection
39 import lsst.afw.image
40 import lsst.afw.geom
41 from lsst.daf.base import PropertyList
42 
43 from . import Defect
44 
45 log = logging.getLogger(__name__)
46 
47 
48 @deprecated(reason="Policy defect files no longer supported (will be removed after v18)",
49  category=FutureWarning)
50 def policyToBadRegionList(policyFile):
51  """Given a Policy file describing a CCD's bad pixels, return a vector of BadRegion::Ptr"""
52 
53  badPixelsPolicy = policy.Policy.createPolicy(policyFile)
54  badPixels = []
55 
56  if badPixelsPolicy.exists("Defects"):
57  d = badPixelsPolicy.getArray("Defects")
58  for reg in d:
59  x0 = reg.get("x0")
60  width = reg.get("width")
61  if not width:
62  x1 = reg.get("x1")
63  width = x1 - x0 - 1
64 
65  y0 = reg.get("y0")
66  if reg.exists("height"):
67  height = reg.get("height")
68  else:
69  y1 = reg.get("y1")
70  height = y1 - y0 - 1
71 
72  bbox = lsst.geom.BoxI(lsst.geom.PointI(x0, y0), lsst.geom.ExtentI(width, height))
73  badPixels.append(Defect(bbox))
74 
75  return badPixels
76 
77 
78 class Defects(collections.abc.MutableSequence):
79  """Collection of `lsst.meas.algorithms.Defect`.
80 
81  Parameters
82  ----------
83  defectList : iterable of `lsst.meas.algorithms.Defect`
84  or `lsst.geom.BoxI`, optional
85  Collections of defects to apply to the image.
86  """
87 
88  _OBSTYPE = "defects"
89  """The calibration type used for ingest."""
90 
91  def __init__(self, defectList=None, metadata=None):
92  self._defects = []
93 
94  if metadata is not None:
95  self._metadata = metadata
96  else:
97  self.setMetadata()
98 
99  if defectList is None:
100  return
101 
102  # Ensure that type checking
103  for d in defectList:
104  self.append(d)
105 
106  def _check_value(self, value):
107  """Check that the supplied value is a `~lsst.meas.algorithms.Defect`
108  or can be converted to one.
109 
110  Parameters
111  ----------
112  value : `object`
113  Value to check.
114 
115  Returns
116  -------
117  new : `~lsst.meas.algorithms.Defect`
118  Either the supplied value or a new object derived from it.
119 
120  Raises
121  ------
122  ValueError
123  Raised if the supplied value can not be converted to
124  `~lsst.meas.algorithms.Defect`
125  """
126  if isinstance(value, Defect):
127  pass
128  elif isinstance(value, lsst.geom.BoxI):
129  value = Defect(value)
130  elif isinstance(value, lsst.geom.PointI):
131  value = Defect(lsst.geom.Box2I(value, lsst.geom.Extent2I(1, 1)))
132  elif isinstance(value, lsst.afw.image.DefectBase):
133  value = Defect(value.getBBox())
134  else:
135  raise ValueError(f"Defects must be of type Defect, BoxI, or PointI, not '{value!r}'")
136  return value
137 
138  def __len__(self):
139  return len(self._defects)
140 
141  def __getitem__(self, index):
142  return self._defects[index]
143 
144  def __setitem__(self, index, value):
145  """Can be given a `~lsst.meas.algorithms.Defect` or a `lsst.geom.BoxI`
146  """
147  self._defects[index] = self._check_value(value)
148 
149  def __iter__(self):
150  return iter(self._defects)
151 
152  def __delitem__(self, index):
153  del self._defects[index]
154 
155  def __eq__(self, other):
156  """Compare if two `Defects` are equal.
157 
158  Two `Defects` are equal if their bounding boxes are equal and in
159  the same order. Metadata content is ignored.
160  """
161  if not isinstance(other, self.__class__):
162  return False
163 
164  # Assume equal if bounding boxes are equal
165  for d1, d2 in zip(self, other):
166  if d1.getBBox() != d2.getBBox():
167  return False
168 
169  return True
170 
171  def __str__(self):
172  return "Defects(" + ",".join(str(d.getBBox()) for d in self) + ")"
173 
174  def insert(self, index, value):
175  self._defects.insert(index, self._check_value(value))
176 
177  def getMetadata(self):
178  """Retrieve metadata associated with these `Defects`.
179 
180  Returns
181  -------
182  meta : `lsst.daf.base.PropertyList`
183  Metadata. The returned `~lsst.daf.base.PropertyList` can be
184  modified by the caller and the changes will be written to
185  external files.
186  """
187  return self._metadata
188 
189  def setMetadata(self, metadata=None):
190  """Store a copy of the supplied metadata with the defects.
191 
192  Parameters
193  ----------
194  metadata : `lsst.daf.base.PropertyList`, optional
195  Metadata to associate with the defects. Will be copied and
196  overwrite existing metadata. If not supplied the existing
197  metadata will be reset.
198  """
199  if metadata is None:
200  self._metadata = PropertyList()
201  else:
202  self._metadata = copy.copy(metadata)
203 
204  # Ensure that we have the obs type required by calibration ingest
205  self._metadata["OBSTYPE"] = self._OBSTYPE
206 
207  def copy(self):
208  """Copy the defects to a new list, creating new defects from the
209  bounding boxes.
210 
211  Returns
212  -------
213  new : `Defects`
214  New list with new `Defect` entries.
215 
216  Notes
217  -----
218  This is not a shallow copy in that new `Defect` instances are
219  created from the original bounding boxes. It's also not a deep
220  copy since the bounding boxes are not recreated.
221  """
222  return self.__class__(d.getBBox() for d in self)
223 
224  def transpose(self):
225  """Make a transposed copy of this defect list.
226 
227  Returns
228  -------
229  retDefectList : `Defects`
230  Transposed list of defects.
231  """
232  retDefectList = self.__class__()
233  for defect in self:
234  bbox = defect.getBBox()
235  dimensions = bbox.getDimensions()
236  nbbox = lsst.geom.Box2I(lsst.geom.Point2I(bbox.getMinY(), bbox.getMinX()),
237  lsst.geom.Extent2I(dimensions[1], dimensions[0]))
238  retDefectList.append(nbbox)
239  return retDefectList
240 
241  def maskPixels(self, maskedImage, maskName="BAD"):
242  """Set mask plane based on these defects.
243 
244  Parameters
245  ----------
246  maskedImage : `lsst.afw.image.MaskedImage`
247  Image to process. Only the mask plane is updated.
248  maskName : str, optional
249  Mask plane name to use.
250  """
251  # mask bad pixels
252  mask = maskedImage.getMask()
253  bitmask = mask.getPlaneBitMask(maskName)
254  for defect in self:
255  bbox = defect.getBBox()
256  lsst.afw.geom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
257 
258  def toTable(self):
259  """Convert defect list to `~lsst.afw.table.BaseCatalog`
260 
261  Returns
262  -------
263  table : `lsst.afw.table.BaseCatalog`
264  Defects in tabular form.
265  """
266  schema = lsst.afw.table.Schema()
267  x = schema.addField("X", type="D", units="pix")
268  y = schema.addField("Y", type="D", units="pix")
269  shape = schema.addField("SHAPE", type="String", size=16)
270  r = schema.addField("R", type="ArrayD", size=2, units="pix")
271  rotang = schema.addField("ROTANG", type="D", units="deg")
272  component = schema.addField("COMPONENT", type="I")
273  table = lsst.afw.table.BaseCatalog(schema)
274 
275  for i, defect in enumerate(self._defects):
276  box = defect.getBBox()
277  record = table.addNew()
278  record.set(x, box.getCenterX())
279  record.set(y, box.getCenterY())
280  record.set(shape, "BOX")
281  record.set(r, np.array([box.getWidth(), box.getHeight()], dtype=np.float64))
282  record.set(rotang, 0.0)
283  record.set(component, i)
284 
285  # Set some metadata in the table (force OBSTYPE to exist)
286  metadata = copy.copy(self.getMetadata())
287  metadata["OBSTYPE"] = self._OBSTYPE
288  table.setMetadata(metadata)
289 
290  return table
291 
292  def writeFits(self, *args):
293  """Write defect list to FITS.
294 
295  Parameters
296  ----------
297  *args
298  Arguments to be forwarded to
299  `lsst.afw.table.BaseCatalog.writeFits`.
300  """
301  table = self.toTable()
302 
303  # Add some additional headers useful for tracking purposes
304  metadata = table.getMetadata()
305  now = datetime.datetime.utcnow()
306  metadata["DATE"] = now.isoformat()
307  metadata["CALIB_CREATION_DATE"] = now.strftime("%Y-%m-%d")
308  metadata["CALIB_CREATION_TIME"] = now.strftime("%T %Z").strip()
309 
310  table.writeFits(*args)
311 
312  @classmethod
313  def fromTable(cls, table):
314  """Construct a `Defects` from the contents of a
315  `~lsst.afw.table.BaseCatalog`.
316 
317  Parameters
318  ----------
319  table : `lsst.afw.table.BaseCatalog`
320  Table with one row per defect.
321 
322  Returns
323  -------
324  defects : `Defects`
325  A `Defects` list.
326  """
327 
328  defectList = []
329 
330  schema = table.getSchema()
331 
332  # Check schema to see which definitions we have
333  if "X" in schema and "Y" in schema and "R" in schema and "SHAPE" in schema:
334  # This is a FITS region style table
335  isFitsRegion = True
336 
337  elif "x0" in schema and "y0" in schema and "width" in schema and "height" in schema:
338  # This is a classic LSST-style defect table
339  isFitsRegion = False
340 
341  else:
342  raise ValueError("Unsupported schema for defects extraction")
343 
344  for r in table:
345  record = r.extract("*")
346 
347  if isFitsRegion:
348  if record["SHAPE"] == "BOX":
349  box = lsst.geom.Box2I.makeCenteredBox(lsst.geom.Point2D(record["X"], record["Y"]),
350  lsst.geom.Extent2I(record["R"]))
351  elif record["SHAPE"] == "POINT":
352  # Handle the case where we have an externally created
353  # FITS file.
354  box = lsst.geom.Point2I(record["X"], record["Y"])
355  else:
356  log.warning("Defect lists can only be defined using BOX not %s", record["SHAPE"])
357  continue
358 
359  elif "x0" in record and "y0" in record and "width" in record and "height" in record:
360  # This is a classic LSST-style defect table
361  box = lsst.geom.Box2I(lsst.geom.Point2I(record["x0"], record["y0"]),
362  lsst.geom.Extent2I(record["width"], record["height"]))
363 
364  defectList.append(box)
365 
366  return cls(defectList)
367 
368  @classmethod
369  def readFits(cls, *args):
370  """Read defect list from FITS table.
371 
372  Parameters
373  ----------
374  *args
375  Arguments to be forwarded to
376  `lsst.afw.table.BaseCatalog.writeFits`.
377 
378  Returns
379  -------
380  defects : `Defects`
381  Defects read from a FITS table.
382  """
383  table = lsst.afw.table.BaseCatalog.readFits(*args)
384  defects = cls.fromTable(table)
385  print(f"Meta: {table.getMetadata()!r}")
386  defects.setMetadata(table.getMetadata())
387  return defects
388 
389  @classmethod
390  def readLsstDefectsFile(cls, filename):
391  """Read defects information from an LSST format text file.
392 
393  Parameters
394  ----------
395  filename : `str`
396  Name of text file containing the defect information.
397 
398  Returns
399  -------
400  defects : `Defects`
401  The defects.
402 
403  Notes
404  -----
405  These defect text files are used as the human readable definitions
406  of defects in calibration data definition repositories. The format
407  is to use four columns defined as follows:
408 
409  x0 : `int`
410  X coordinate of bottom left corner of box.
411  y0 : `int`
412  Y coordinate of bottom left corner of box.
413  width : `int`
414  X extent of the box.
415  height : `int`
416  Y extent of the box.
417  """
418  # Use loadtxt so that ValueError is thrown if the file contains a
419  # non-integer value. genfromtxt converts bad values to -1.
420  defect_array = np.loadtxt(filename,
421  dtype=[("x0", "int"), ("y0", "int"),
422  ("x_extent", "int"), ("y_extent", "int")])
423 
424  return cls(lsst.geom.Box2I(lsst.geom.Point2I(row["x0"], row["y0"]),
425  lsst.geom.Extent2I(row["x_extent"], row["y_extent"]))
426  for row in defect_array)
427 
428  @classmethod
429  def fromFootprintList(cls, fpList):
430  """Compute a defect list from a footprint list, optionally growing
431  the footprints.
432 
433  Parameters
434  ----------
435  fpList : `list` of `lsst.afw.detection.Footprint`
436  Footprint list to process.
437 
438  Returns
439  -------
440  defects : `Defects`
441  List of defects.
442  """
443  return cls(itertools.chain.from_iterable(lsst.afw.detection.footprintToBBoxList(fp)
444  for fp in fpList))
445 
446  @classmethod
447  def fromMask(cls, maskedImage, maskName):
448  """Compute a defect list from a specified mask plane.
449 
450  Parameters
451  ----------
452  maskedImage : `lsst.afw.image.MaskedImage`
453  Image to process.
454  maskName : `str` or `list`
455  Mask plane name, or list of names to convert.
456 
457  Returns
458  -------
459  defects : `Defects`
460  Defect list constructed from masked pixels.
461  """
462  mask = maskedImage.getMask()
463  thresh = lsst.afw.detection.Threshold(mask.getPlaneBitMask(maskName),
464  lsst.afw.detection.Threshold.BITMASK)
465  fpList = lsst.afw.detection.FootprintSet(mask, thresh).getFootprints()
466  return cls.fromFootprintList(fpList)
def maskPixels(self, maskedImage, maskName="BAD")
Definition: defects.py:241
def setMetadata(self, metadata=None)
Definition: defects.py:189
def __setitem__(self, index, value)
Definition: defects.py:144
def __init__(self, defectList=None, metadata=None)
Definition: defects.py:91
def policyToBadRegionList(policyFile)
Definition: defects.py:50
def readLsstDefectsFile(cls, filename)
Definition: defects.py:390
std::vector< lsst::geom::Box2I > footprintToBBoxList(Footprint const &footprint)
def insert(self, index, value)
Definition: defects.py:174
static Box2I makeCenteredBox(Point2D const &center, Extent const &size)
def fromMask(cls, maskedImage, maskName)
Definition: defects.py:447