Coverage for python/lsst/afw/table/_base.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import numpy as np
23from lsst.utils import continueClass, TemplateMeta
24from ._table import BaseRecord, BaseCatalog
25from ._schema import Key
28__all__ = ["Catalog"]
31@continueClass # noqa: F811
32class BaseRecord:
34 def extract(self, *patterns, **kwargs):
35 """Extract a dictionary of {<name>: <field-value>} in which the field names
36 match the given shell-style glob pattern(s).
38 Any number of glob patterns may be passed; the result will be the union of all
39 the result of each glob considered separately.
41 Parameters
42 ----------
43 items : `dict`
44 The result of a call to self.schema.extract(); this will be used
45 instead of doing any new matching, and allows the pattern matching
46 to be reused to extract values from multiple records. This
47 keyword is incompatible with any position arguments and the regex,
48 sub, and ordered keyword arguments.
49 split : `bool`
50 If `True`, fields with named subfields (e.g. points) will be split
51 into separate items in the dict; instead of {"point":
52 lsst.geom.Point2I(2,3)}, for instance, you'd get {"point.x":
53 2, "point.y": 3}. Default is `False`.
54 regex : `str` or `re` pattern object
55 A regular expression to be used in addition to any glob patterns
56 passed as positional arguments. Note that this will be compared
57 with re.match, not re.search.
58 sub : `str`
59 A replacement string (see `re.MatchObject.expand`) used to set the
60 dictionary keys of any fields matched by regex.
61 ordered : `bool`
62 If `True`, a `collections.OrderedDict` will be returned instead of
63 a standard dict, with the order corresponding to the definition
64 order of the `Schema`. Default is `False`.
65 """
66 d = kwargs.pop("items", None)
67 split = kwargs.pop("split", False)
68 if d is None:
69 d = self.schema.extract(*patterns, **kwargs).copy()
70 elif kwargs:
71 kwargsStr = ", ".join(kwargs.keys())
72 raise ValueError(f"Unrecognized keyword arguments for extract: {kwargsStr}")
73 # must use list because we might be adding/deleting elements
74 for name, schemaItem in list(d.items()):
75 key = schemaItem.key
76 if split and key.HAS_NAMED_SUBFIELDS:
77 for subname, subkey in zip(key.subfields, key.subkeys):
78 d[f"{name}.{subname}"] = self.get(subkey)
79 del d[name]
80 else:
81 d[name] = self.get(schemaItem.key)
82 return d
84 def __repr__(self):
85 return f"{type(self)}\n{self}"
88class Catalog(metaclass=TemplateMeta):
90 def getColumnView(self):
91 self._columns = self._getColumnView()
92 return self._columns
94 def __getColumns(self):
95 if not hasattr(self, "_columns") or self._columns is None:
96 self._columns = self._getColumnView()
97 return self._columns
98 columns = property(__getColumns, doc="a column view of the catalog")
100 def __getitem__(self, key):
101 """Return the record at index key if key is an integer,
102 return a column if `key` is a string field name or Key,
103 or return a subset of the catalog if key is a slice
104 or boolean NumPy array.
105 """
106 if type(key) is slice:
107 (start, stop, step) = (key.start, key.stop, key.step)
108 if step is None:
109 step = 1
110 if start is None:
111 start = 0
112 if stop is None:
113 stop = len(self)
114 return self.subset(start, stop, step)
115 elif isinstance(key, np.ndarray):
116 if key.dtype == bool:
117 return self.subset(key)
118 raise RuntimeError(f"Unsupported array type for indexing non-contiguous Catalog: {key.dtype}")
119 elif isinstance(key, Key) or isinstance(key, str):
120 if not self.isContiguous():
121 if isinstance(key, str):
122 key = self.schema[key].asKey()
123 array = self._getitem_(key)
124 # This array doesn't share memory with the Catalog, so don't let it be modified by
125 # the user who thinks that the Catalog itself is being modified.
126 # Just be aware that this array can only be passed down to C++ as an ndarray::Array<T const>
127 # instead of an ordinary ndarray::Array<T>. If pybind isn't letting it down into C++,
128 # you may have left off the 'const' in the definition.
129 array.flags.writeable = False
130 return array
131 return self.columns[key]
132 else:
133 return self._getitem_(key)
135 def __setitem__(self, key, value):
136 """If ``key`` is an integer, set ``catalog[key]`` to
137 ``value``. Otherwise select column ``key`` and set it to
138 ``value``.
139 """
140 self._columns = None
141 if isinstance(key, Key) or isinstance(key, str):
142 self.columns[key] = value
143 else:
144 return self.set(key, value)
146 def __delitem__(self, key):
147 self._columns = None
148 if isinstance(key, slice):
149 self._delslice_(key)
150 else:
151 self._delitem_(key)
153 def append(self, record):
154 self._columns = None
155 self._append(record)
157 def insert(self, key, value):
158 self._columns = None
159 self._insert(key, value)
161 def clear(self):
162 self._columns = None
163 self._clear()
165 def addNew(self):
166 self._columns = None
167 return self._addNew()
169 def cast(self, type_, deep=False):
170 """Return a copy of the catalog with the given type.
172 Parameters
173 ----------
174 type_ :
175 Type of catalog to return.
176 deep : `bool`, optional
177 If `True`, clone the table and deep copy all records.
179 Returns
180 -------
181 copy :
182 Copy of catalog with the requested type.
183 """
184 if deep:
185 table = self.table.clone()
186 table.preallocate(len(self))
187 else:
188 table = self.table
189 copy = type_(table)
190 copy.extend(self, deep=deep)
191 return copy
193 def copy(self, deep=False):
194 """
195 Copy a catalog (default is not a deep copy).
196 """
197 return self.cast(type(self), deep)
199 def extend(self, iterable, deep=False, mapper=None):
200 """Append all records in the given iterable to the catalog.
202 Parameters
203 ----------
204 iterable :
205 Any Python iterable containing records.
206 deep : `bool`, optional
207 If `True`, the records will be deep-copied; ignored if
208 mapper is not `None` (that always implies `True`).
209 mapper : `lsst.afw.table.schemaMapper.SchemaMapper`, optional
210 Used to translate records.
211 """
212 self._columns = None
213 # We can't use isinstance here, because the SchemaMapper symbol isn't available
214 # when this code is part of a subclass of Catalog in another package.
215 if type(deep).__name__ == "SchemaMapper":
216 mapper = deep
217 deep = None
218 if isinstance(iterable, type(self)):
219 if mapper is not None:
220 self._extend(iterable, mapper)
221 else:
222 self._extend(iterable, deep)
223 else:
224 for record in iterable:
225 if mapper is not None:
226 self._append(self.table.copyRecord(record, mapper))
227 elif deep:
228 self._append(self.table.copyRecord(record))
229 else:
230 self._append(record)
232 def __reduce__(self):
233 import lsst.afw.fits
234 return lsst.afw.fits.reduceToFits(self)
236 def asAstropy(self, cls=None, copy=False, unviewable="copy"):
237 """Return an astropy.table.Table (or subclass thereof) view into this catalog.
239 Parameters
240 ----------
241 cls :
242 Table subclass to use; `None` implies `astropy.table.Table`
243 itself. Use `astropy.table.QTable` to get Quantity columns.
244 copy : bool, optional
245 If `True`, copy data from the LSST catalog to the astropy
246 table. Not copying is usually faster, but can keep memory
247 from being freed if columns are later removed from the
248 Astropy view.
249 unviewable : `str`, optional
250 One of the following options (which is ignored if
251 copy=`True` ), indicating how to handle field types (`str`
252 and `Flag`) for which views cannot be constructed:
253 - 'copy' (default): copy only the unviewable fields.
254 - 'raise': raise ValueError if unviewable fields are present.
255 - 'skip': do not include unviewable fields in the Astropy Table.
257 Returns
258 -------
259 cls : `astropy.table.Table`
260 Astropy view into the catalog.
262 Raises
263 ------
264 ValueError
265 Raised if the `unviewable` option is not a known value, or
266 if the option is 'raise' and an uncopyable field is found.
268 """
269 import astropy.table
270 if cls is None:
271 cls = astropy.table.Table
272 if unviewable not in ("copy", "raise", "skip"):
273 raise ValueError(
274 f"'unviewable'={unviewable!r} must be one of 'copy', 'raise', or 'skip'")
275 ps = self.getMetadata()
276 meta = ps.toOrderedDict() if ps is not None else None
277 columns = []
278 items = self.schema.extract("*", ordered=True)
279 for name, item in items.items():
280 key = item.key
281 unit = item.field.getUnits() or None # use None instead of "" when empty
282 if key.getTypeString() == "String":
283 if not copy:
284 if unviewable == "raise":
285 raise ValueError("Cannot extract string "
286 "unless copy=True or unviewable='copy' or 'skip'.")
287 elif unviewable == "skip":
288 continue
289 data = np.zeros(
290 len(self), dtype=np.dtype((str, key.getSize())))
291 for i, record in enumerate(self):
292 data[i] = record.get(key)
293 elif key.getTypeString() == "Flag":
294 if not copy:
295 if unviewable == "raise":
296 raise ValueError("Cannot extract packed bit columns "
297 "unless copy=True or unviewable='copy' or 'skip'.")
298 elif unviewable == "skip":
299 continue
300 data = self.columns.get_bool_array(key)
301 elif key.getTypeString() == "Angle":
302 data = self.columns.get(key)
303 unit = "radian"
304 if copy:
305 data = data.copy()
306 elif "Array" in key.getTypeString() and key.isVariableLength():
307 # Can't get columns for variable-length array fields.
308 if unviewable == "raise":
309 raise ValueError("Cannot extract variable-length array fields unless unviewable='skip'.")
310 elif unviewable == "skip" or unviewable == "copy":
311 continue
312 else:
313 data = self.columns.get(key)
314 if copy:
315 data = data.copy()
316 columns.append(
317 astropy.table.Column(
318 data,
319 name=name,
320 unit=unit,
321 description=item.field.getDoc()
322 )
323 )
324 return cls(columns, meta=meta, copy=False)
326 def __dir__(self):
327 """
328 This custom dir is necessary due to the custom getattr below.
329 Without it, not all of the methods available are returned with dir.
330 See DM-7199.
331 """
332 def recursive_get_class_dir(cls):
333 """
334 Return a set containing the names of all methods
335 for a given class *and* all of its subclasses.
336 """
337 result = set()
338 if cls.__bases__:
339 for subcls in cls.__bases__:
340 result |= recursive_get_class_dir(subcls)
341 result |= set(cls.__dict__.keys())
342 return result
343 return sorted(set(dir(self.columns)) | set(dir(self.table))
344 | recursive_get_class_dir(type(self)) | set(self.__dict__.keys()))
346 def __getattr__(self, name):
347 # Catalog forwards unknown method calls to its table and column view
348 # for convenience. (Feature requested by RHL; complaints about magic
349 # should be directed to him.)
350 if name == "_columns":
351 self._columns = None
352 return None
353 try:
354 return getattr(self.table, name)
355 except AttributeError:
356 return getattr(self.columns, name)
358 def __str__(self):
359 if self.isContiguous():
360 return str(self.asAstropy())
361 else:
362 fields = ' '.join(x.field.getName() for x in self.schema)
363 return f"Non-contiguous afw.Catalog of {len(self)} rows.\ncolumns: {fields}"
365 def __repr__(self):
366 return "%s\n%s" % (type(self), self)
369Catalog.register("Base", BaseCatalog)