lsst.afw  19.0.0-23-g9b026a6e0
_base.py
Go to the documentation of this file.
1 # This file is part of afw.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 import numpy as np
22 
23 from lsst.utils import continueClass, TemplateMeta
24 from ._table import BaseRecord, BaseCatalog
25 from ._schema import Key
26 
27 
28 __all__ = ["Catalog"]
29 
30 
31 @continueClass # noqa: F811
32 class BaseRecord:
33 
34  def extract(self, *patterns, **kwargs):
35  """Extract a dictionary of {<name>: <field-value>} in which the field names
36  match the given shell-style glob pattern(s).
37 
38  Any number of glob patterns may be passed; the result will be the union of all
39  the result of each glob considered separately.
40 
41  Parameters
42  ----------
43  items : `dict`
44  The result of a call to self.schema.extract(); this will be used
45  instead of doing any new matching, and allows the pattern matching
46  to be reused to extract values from multiple records. This
47  keyword is incompatible with any position arguments and the regex,
48  sub, and ordered keyword arguments.
49  split : `bool`
50  If `True`, fields with named subfields (e.g. points) will be split
51  into separate items in the dict; instead of {"point":
52  lsst.geom.Point2I(2,3)}, for instance, you'd get {"point.x":
53  2, "point.y": 3}. Default is `False`.
54  regex : `str` or `re` pattern object
55  A regular expression to be used in addition to any glob patterns
56  passed as positional arguments. Note that this will be compared
57  with re.match, not re.search.
58  sub : `str`
59  A replacement string (see `re.MatchObject.expand`) used to set the
60  dictionary keys of any fields matched by regex.
61  ordered : `bool`
62  If `True`, a `collections.OrderedDict` will be returned instead of
63  a standard dict, with the order corresponding to the definition
64  order of the `Schema`. Default is `False`.
65  """
66  d = kwargs.pop("items", None)
67  split = kwargs.pop("split", False)
68  if d is None:
69  d = self.schema.extract(*patterns, **kwargs).copy()
70  elif kwargs:
71  kwargsStr = ", ".join(kwargs.keys())
72  raise ValueError(f"Unrecognized keyword arguments for extract: {kwargsStr}")
73  # must use list because we might be adding/deleting elements
74  for name, schemaItem in list(d.items()):
75  key = schemaItem.key
76  if split and key.HAS_NAMED_SUBFIELDS:
77  for subname, subkey in zip(key.subfields, key.subkeys):
78  d[f"{name}.{subname}"] = self.get(subkey)
79  del d[name]
80  else:
81  d[name] = self.get(schemaItem.key)
82  return d
83 
84  def __repr__(self):
85  return f"{type(self)}\n{self}"
86 
87 
88 class Catalog(metaclass=TemplateMeta):
89 
90  def getColumnView(self):
91  self._columns = self._getColumnView()
92  return self._columns
93 
94  def __getColumns(self):
95  if not hasattr(self, "_columns") or self._columns is None:
96  self._columns = self._getColumnView()
97  return self._columns
98  columns = property(__getColumns, doc="a column view of the catalog")
99 
100  def __getitem__(self, key):
101  """Return the record at index key if key is an integer,
102  return a column if `key` is a string field name or Key,
103  or return a subset of the catalog if key is a slice
104  or boolean NumPy array.
105  """
106  if type(key) is slice:
107  (start, stop, step) = (key.start, key.stop, key.step)
108  if step is None:
109  step = 1
110  if start is None:
111  start = 0
112  if stop is None:
113  stop = len(self)
114  return self.subset(start, stop, step)
115  elif isinstance(key, np.ndarray):
116  if key.dtype == bool:
117  return self.subset(key)
118  raise RuntimeError(f"Unsupported array type for indexing non-contiguous Catalog: {key.dtype}")
119  elif isinstance(key, Key) or isinstance(key, str):
120  if not self.isContiguous():
121  if isinstance(key, str):
122  key = self.schema[key].asKey()
123  array = self._getitem_(key)
124  # This array doesn't share memory with the Catalog, so don't let it be modified by
125  # the user who thinks that the Catalog itself is being modified.
126  # Just be aware that this array can only be passed down to C++ as an ndarray::Array<T const>
127  # instead of an ordinary ndarray::Array<T>. If pybind isn't letting it down into C++,
128  # you may have left off the 'const' in the definition.
129  array.flags.writeable = False
130  return array
131  return self.columns[key]
132  else:
133  return self._getitem_(key)
134 
135  def __setitem__(self, key, value):
136  """If ``key`` is an integer, set ``catalog[key]`` to
137  ``value``. Otherwise select column ``key`` and set it to
138  ``value``.
139  """
140  self._columns = None
141  if isinstance(key, Key) or isinstance(key, str):
142  self.columns[key] = value
143  else:
144  return self.set(key, value)
145 
146  def __delitem__(self, key):
147  self._columns = None
148  if isinstance(key, slice):
149  self._delslice_(key)
150  else:
151  self._delitem_(key)
152 
153  def append(self, record):
154  self._columns = None
155  self._append(record)
156 
157  def insert(self, key, value):
158  self._columns = None
159  self._insert(key, value)
160 
161  def clear(self):
162  self._columns = None
163  self._clear()
164 
165  def addNew(self):
166  self._columns = None
167  return self._addNew()
168 
169  def cast(self, type_, deep=False):
170  """Return a copy of the catalog with the given type.
171 
172  Parameters
173  ----------
174  type_ :
175  Type of catalog to return.
176  deep : `bool`, optional
177  If `True`, clone the table and deep copy all records.
178 
179  Returns
180  -------
181  copy :
182  Copy of catalog with the requested type.
183  """
184  if deep:
185  table = self.table.clone()
186  table.preallocate(len(self))
187  else:
188  table = self.table
189  copy = type_(table)
190  copy.extend(self, deep=deep)
191  return copy
192 
193  def copy(self, deep=False):
194  """
195  Copy a catalog (default is not a deep copy).
196  """
197  return self.cast(type(self), deep)
198 
199  def extend(self, iterable, deep=False, mapper=None):
200  """Append all records in the given iterable to the catalog.
201 
202  Parameters
203  ----------
204  iterable :
205  Any Python iterable containing records.
206  deep : `bool`, optional
207  If `True`, the records will be deep-copied; ignored if
208  mapper is not `None` (that always implies `True`).
209  mapper : `lsst.afw.table.schemaMapper.SchemaMapper`, optional
210  Used to translate records.
211  """
212  self._columns = None
213  # We can't use isinstance here, because the SchemaMapper symbol isn't available
214  # when this code is part of a subclass of Catalog in another package.
215  if type(deep).__name__ == "SchemaMapper":
216  mapper = deep
217  deep = None
218  if isinstance(iterable, type(self)):
219  if mapper is not None:
220  self._extend(iterable, mapper)
221  else:
222  self._extend(iterable, deep)
223  else:
224  for record in iterable:
225  if mapper is not None:
226  self._append(self.table.copyRecord(record, mapper))
227  elif deep:
228  self._append(self.table.copyRecord(record))
229  else:
230  self._append(record)
231 
232  def __reduce__(self):
233  import lsst.afw.fits
234  return lsst.afw.fits.reduceToFits(self)
235 
236  def asAstropy(self, cls=None, copy=False, unviewable="copy"):
237  """Return an astropy.table.Table (or subclass thereof) view into this catalog.
238 
239  Parameters
240  ----------
241  cls :
242  Table subclass to use; `None` implies `astropy.table.Table`
243  itself. Use `astropy.table.QTable` to get Quantity columns.
244  copy : bool, optional
245  If `True`, copy data from the LSST catalog to the astropy
246  table. Not copying is usually faster, but can keep memory
247  from being freed if columns are later removed from the
248  Astropy view.
249  unviewable : `str`, optional
250  One of the following options (which is ignored if
251  copy=`True` ), indicating how to handle field types (`str`
252  and `Flag`) for which views cannot be constructed:
253  - 'copy' (default): copy only the unviewable fields.
254  - 'raise': raise ValueError if unviewable fields are present.
255  - 'skip': do not include unviewable fields in the Astropy Table.
256 
257  Returns
258  -------
259  cls : `astropy.table.Table`
260  Astropy view into the catalog.
261 
262  Raises
263  ------
264  ValueError
265  Raised if the `unviewable` option is not a known value, or
266  if the option is 'raise' and an uncopyable field is found.
267 
268  """
269  import astropy.table
270  if cls is None:
271  cls = astropy.table.Table
272  if unviewable not in ("copy", "raise", "skip"):
273  raise ValueError(
274  f"'unviewable'={unviewable!r} must be one of 'copy', 'raise', or 'skip'")
275  ps = self.getMetadata()
276  meta = ps.toOrderedDict() if ps is not None else None
277  columns = []
278  items = self.schema.extract("*", ordered=True)
279  for name, item in items.items():
280  key = item.key
281  unit = item.field.getUnits() or None # use None instead of "" when empty
282  if key.getTypeString() == "String":
283  if not copy:
284  if unviewable == "raise":
285  raise ValueError("Cannot extract string "
286  "unless copy=True or unviewable='copy' or 'skip'.")
287  elif unviewable == "skip":
288  continue
289  data = np.zeros(
290  len(self), dtype=np.dtype((str, key.getSize())))
291  for i, record in enumerate(self):
292  data[i] = record.get(key)
293  elif key.getTypeString() == "Flag":
294  if not copy:
295  if unviewable == "raise":
296  raise ValueError("Cannot extract packed bit columns "
297  "unless copy=True or unviewable='copy' or 'skip'.")
298  elif unviewable == "skip":
299  continue
300  data = self.columns.get_bool_array(key)
301  elif key.getTypeString() == "Angle":
302  data = self.columns.get(key)
303  unit = "radian"
304  if copy:
305  data = data.copy()
306  elif "Array" in key.getTypeString() and key.isVariableLength():
307  # Can't get columns for variable-length array fields.
308  if unviewable == "raise":
309  raise ValueError("Cannot extract variable-length array fields unless unviewable='skip'.")
310  elif unviewable == "skip" or unviewable == "copy":
311  continue
312  else:
313  data = self.columns.get(key)
314  if copy:
315  data = data.copy()
316  columns.append(
317  astropy.table.Column(
318  data,
319  name=name,
320  unit=unit,
321  description=item.field.getDoc()
322  )
323  )
324  return cls(columns, meta=meta, copy=False)
325 
326  def __dir__(self):
327  """
328  This custom dir is necessary due to the custom getattr below.
329  Without it, not all of the methods available are returned with dir.
330  See DM-7199.
331  """
332  def recursive_get_class_dir(cls):
333  """
334  Return a set containing the names of all methods
335  for a given class *and* all of its subclasses.
336  """
337  result = set()
338  if cls.__bases__:
339  for subcls in cls.__bases__:
340  result |= recursive_get_class_dir(subcls)
341  result |= set(cls.__dict__.keys())
342  return result
343  return sorted(set(dir(self.columns)) | set(dir(self.table))
344  | recursive_get_class_dir(type(self)) | set(self.__dict__.keys()))
345 
346  def __getattr__(self, name):
347  # Catalog forwards unknown method calls to its table and column view
348  # for convenience. (Feature requested by RHL; complaints about magic
349  # should be directed to him.)
350  if name == "_columns":
351  self._columns = None
352  return None
353  try:
354  return getattr(self.table, name)
355  except AttributeError:
356  return getattr(self.columns, name)
357 
358  def __str__(self):
359  if self.isContiguous():
360  return str(self.asAstropy())
361  else:
362  fields = ' '.join(x.field.getName() for x in self.schema)
363  return f"Non-contiguous afw.Catalog of {len(self)} rows.\ncolumns: {fields}"
364 
365  def __repr__(self):
366  return "%s\n%s" % (type(self), self)
367 
368 
369 Catalog.register("Base", BaseCatalog)
lsst::afw::table._base.BaseRecord.extract
def extract(self, *patterns, **kwargs)
Definition: _base.py:34
lsst::afw::table._base.Catalog.copy
def copy(self, deep=False)
Definition: _base.py:193
lsst::afw::table._base.Catalog.__getattr__
def __getattr__(self, name)
Definition: _base.py:346
lsst::afw::table._base.Catalog.__setitem__
def __setitem__(self, key, value)
Definition: _base.py:135
lsst::afw::table._base.BaseRecord
Definition: _base.py:32
lsst::afw::table._base.Catalog
Definition: _base.py:88
list
daf::base::PropertyList * list
Definition: fits.cc:913
lsst::afw::geom.transform.transformContinued.cls
cls
Definition: transformContinued.py:33
lsst::afw::table._base.Catalog.insert
def insert(self, key, value)
Definition: _base.py:157
lsst::afw::table._base.Catalog.columns
columns
Definition: _base.py:98
lsst::afw::table._base.Catalog.clear
def clear(self)
Definition: _base.py:161
lsst::afw::table._base.Catalog.__str__
def __str__(self)
Definition: _base.py:358
set
daf::base::PropertySet * set
Definition: fits.cc:912
lsst::afw::table._base.Catalog.__dir__
def __dir__(self)
Definition: _base.py:326
lsst::afw::table._base.Catalog.__delitem__
def __delitem__(self, key)
Definition: _base.py:146
lsst::afw::table._base.Catalog._columns
_columns
Definition: _base.py:91
lsst::utils
lsst::afw::table._base.Catalog.cast
def cast(self, type_, deep=False)
Definition: _base.py:169
lsst::afw::table._base.Catalog.__reduce__
def __reduce__(self)
Definition: _base.py:232
lsst::afw::table._base.Catalog.__repr__
def __repr__(self)
Definition: _base.py:365
lsst::afw::fits
Definition: fits.h:31
lsst::afw::table._base.Catalog.__getitem__
def __getitem__(self, key)
Definition: _base.py:100
type
table::Key< int > type
Definition: Detector.cc:163
lsst::afw::table._base.Catalog.asAstropy
def asAstropy(self, cls=None, copy=False, unviewable="copy")
Definition: _base.py:236
lsst::afw::table._base.Catalog.getColumnView
def getColumnView(self)
Definition: _base.py:90
lsst::afw::table._base.Catalog.extend
def extend(self, iterable, deep=False, mapper=None)
Definition: _base.py:199
lsst::afw::table._base.BaseRecord.__repr__
def __repr__(self)
Definition: _base.py:84
lsst::afw::image.slicing.clone
clone
Definition: slicing.py:257
lsst::afw::table._base.Catalog.addNew
def addNew(self)
Definition: _base.py:165
lsst::afw::table._base.Catalog.append
def append(self, record)
Definition: _base.py:153