Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import numpy as np 

22 

23from lsst.utils import continueClass, TemplateMeta 

24from ._table import BaseRecord, BaseCatalog 

25from ._schema import Key 

26 

27 

28__all__ = ["Catalog"] 

29 

30 

31@continueClass # noqa: F811 

32class BaseRecord: 

33 

34 def extract(self, *patterns, **kwargs): 

35 """Extract a dictionary of {<name>: <field-value>} in which the field names 

36 match the given shell-style glob pattern(s). 

37 

38 Any number of glob patterns may be passed; the result will be the union of all 

39 the result of each glob considered separately. 

40 

41 Parameters 

42 ---------- 

43 items : `dict` 

44 The result of a call to self.schema.extract(); this will be used 

45 instead of doing any new matching, and allows the pattern matching 

46 to be reused to extract values from multiple records. This 

47 keyword is incompatible with any position arguments and the regex, 

48 sub, and ordered keyword arguments. 

49 split : `bool` 

50 If `True`, fields with named subfields (e.g. points) will be split 

51 into separate items in the dict; instead of {"point": 

52 lsst.geom.Point2I(2,3)}, for instance, you'd get {"point.x": 

53 2, "point.y": 3}. Default is `False`. 

54 regex : `str` or `re` pattern object 

55 A regular expression to be used in addition to any glob patterns 

56 passed as positional arguments. Note that this will be compared 

57 with re.match, not re.search. 

58 sub : `str` 

59 A replacement string (see `re.MatchObject.expand`) used to set the 

60 dictionary keys of any fields matched by regex. 

61 ordered : `bool` 

62 If `True`, a `collections.OrderedDict` will be returned instead of 

63 a standard dict, with the order corresponding to the definition 

64 order of the `Schema`. Default is `False`. 

65 """ 

66 d = kwargs.pop("items", None) 

67 split = kwargs.pop("split", False) 

68 if d is None: 

69 d = self.schema.extract(*patterns, **kwargs).copy() 

70 elif kwargs: 

71 kwargsStr = ", ".join(kwargs.keys()) 

72 raise ValueError(f"Unrecognized keyword arguments for extract: {kwargsStr}") 

73 # must use list because we might be adding/deleting elements 

74 for name, schemaItem in list(d.items()): 

75 key = schemaItem.key 

76 if split and key.HAS_NAMED_SUBFIELDS: 

77 for subname, subkey in zip(key.subfields, key.subkeys): 

78 d[f"{name}.{subname}"] = self.get(subkey) 

79 del d[name] 

80 else: 

81 d[name] = self.get(schemaItem.key) 

82 return d 

83 

84 def __repr__(self): 

85 return f"{type(self)}\n{self}" 

86 

87 

88class Catalog(metaclass=TemplateMeta): 

89 

90 def getColumnView(self): 

91 self._columns = self._getColumnView() 

92 return self._columns 

93 

94 def __getColumns(self): 

95 if not hasattr(self, "_columns") or self._columns is None: 

96 self._columns = self._getColumnView() 

97 return self._columns 

98 columns = property(__getColumns, doc="a column view of the catalog") 

99 

100 def __getitem__(self, key): 

101 """Return the record at index key if key is an integer, 

102 return a column if `key` is a string field name or Key, 

103 or return a subset of the catalog if key is a slice 

104 or boolean NumPy array. 

105 """ 

106 if type(key) is slice: 

107 (start, stop, step) = (key.start, key.stop, key.step) 

108 if step is None: 

109 step = 1 

110 if start is None: 

111 start = 0 

112 if stop is None: 

113 stop = len(self) 

114 return self.subset(start, stop, step) 

115 elif isinstance(key, np.ndarray): 

116 if key.dtype == bool: 

117 return self.subset(key) 

118 raise RuntimeError(f"Unsupported array type for indexing non-contiguous Catalog: {key.dtype}") 

119 elif isinstance(key, Key) or isinstance(key, str): 

120 if not self.isContiguous(): 

121 if isinstance(key, str): 

122 key = self.schema[key].asKey() 

123 array = self._getitem_(key) 

124 # This array doesn't share memory with the Catalog, so don't let it be modified by 

125 # the user who thinks that the Catalog itself is being modified. 

126 # Just be aware that this array can only be passed down to C++ as an ndarray::Array<T const> 

127 # instead of an ordinary ndarray::Array<T>. If pybind isn't letting it down into C++, 

128 # you may have left off the 'const' in the definition. 

129 array.flags.writeable = False 

130 return array 

131 return self.columns[key] 

132 else: 

133 return self._getitem_(key) 

134 

135 def __setitem__(self, key, value): 

136 """If ``key`` is an integer, set ``catalog[key]`` to 

137 ``value``. Otherwise select column ``key`` and set it to 

138 ``value``. 

139 """ 

140 self._columns = None 

141 if isinstance(key, Key) or isinstance(key, str): 

142 self.columns[key] = value 

143 else: 

144 return self.set(key, value) 

145 

146 def __delitem__(self, key): 

147 self._columns = None 

148 if isinstance(key, slice): 

149 self._delslice_(key) 

150 else: 

151 self._delitem_(key) 

152 

153 def append(self, record): 

154 self._columns = None 

155 self._append(record) 

156 

157 def insert(self, key, value): 

158 self._columns = None 

159 self._insert(key, value) 

160 

161 def clear(self): 

162 self._columns = None 

163 self._clear() 

164 

165 def addNew(self): 

166 self._columns = None 

167 return self._addNew() 

168 

169 def cast(self, type_, deep=False): 

170 """Return a copy of the catalog with the given type. 

171 

172 Parameters 

173 ---------- 

174 type_ : 

175 Type of catalog to return. 

176 deep : `bool`, optional 

177 If `True`, clone the table and deep copy all records. 

178 

179 Returns 

180 ------- 

181 copy : 

182 Copy of catalog with the requested type. 

183 """ 

184 if deep: 

185 table = self.table.clone() 

186 table.preallocate(len(self)) 

187 else: 

188 table = self.table 

189 copy = type_(table) 

190 copy.extend(self, deep=deep) 

191 return copy 

192 

193 def copy(self, deep=False): 

194 """ 

195 Copy a catalog (default is not a deep copy). 

196 """ 

197 return self.cast(type(self), deep) 

198 

199 def extend(self, iterable, deep=False, mapper=None): 

200 """Append all records in the given iterable to the catalog. 

201 

202 Parameters 

203 ---------- 

204 iterable : 

205 Any Python iterable containing records. 

206 deep : `bool`, optional 

207 If `True`, the records will be deep-copied; ignored if 

208 mapper is not `None` (that always implies `True`). 

209 mapper : `lsst.afw.table.schemaMapper.SchemaMapper`, optional 

210 Used to translate records. 

211 """ 

212 self._columns = None 

213 # We can't use isinstance here, because the SchemaMapper symbol isn't available 

214 # when this code is part of a subclass of Catalog in another package. 

215 if type(deep).__name__ == "SchemaMapper": 

216 mapper = deep 

217 deep = None 

218 if isinstance(iterable, type(self)): 

219 if mapper is not None: 

220 self._extend(iterable, mapper) 

221 else: 

222 self._extend(iterable, deep) 

223 else: 

224 for record in iterable: 

225 if mapper is not None: 

226 self._append(self.table.copyRecord(record, mapper)) 

227 elif deep: 

228 self._append(self.table.copyRecord(record)) 

229 else: 

230 self._append(record) 

231 

232 def __reduce__(self): 

233 import lsst.afw.fits 

234 return lsst.afw.fits.reduceToFits(self) 

235 

236 def asAstropy(self, cls=None, copy=False, unviewable="copy"): 

237 """Return an astropy.table.Table (or subclass thereof) view into this catalog. 

238 

239 Parameters 

240 ---------- 

241 cls : 

242 Table subclass to use; `None` implies `astropy.table.Table` 

243 itself. Use `astropy.table.QTable` to get Quantity columns. 

244 copy : bool, optional 

245 If `True`, copy data from the LSST catalog to the astropy 

246 table. Not copying is usually faster, but can keep memory 

247 from being freed if columns are later removed from the 

248 Astropy view. 

249 unviewable : `str`, optional 

250 One of the following options (which is ignored if 

251 copy=`True` ), indicating how to handle field types (`str` 

252 and `Flag`) for which views cannot be constructed: 

253 - 'copy' (default): copy only the unviewable fields. 

254 - 'raise': raise ValueError if unviewable fields are present. 

255 - 'skip': do not include unviewable fields in the Astropy Table. 

256 

257 Returns 

258 ------- 

259 cls : `astropy.table.Table` 

260 Astropy view into the catalog. 

261 

262 Raises 

263 ------ 

264 ValueError 

265 Raised if the `unviewable` option is not a known value, or 

266 if the option is 'raise' and an uncopyable field is found. 

267 

268 """ 

269 import astropy.table 

270 if cls is None: 

271 cls = astropy.table.Table 

272 if unviewable not in ("copy", "raise", "skip"): 

273 raise ValueError( 

274 f"'unviewable'={unviewable!r} must be one of 'copy', 'raise', or 'skip'") 

275 ps = self.getMetadata() 

276 meta = ps.toOrderedDict() if ps is not None else None 

277 columns = [] 

278 items = self.schema.extract("*", ordered=True) 

279 for name, item in items.items(): 

280 key = item.key 

281 unit = item.field.getUnits() or None # use None instead of "" when empty 

282 if key.getTypeString() == "String": 

283 if not copy: 

284 if unviewable == "raise": 

285 raise ValueError("Cannot extract string " 

286 "unless copy=True or unviewable='copy' or 'skip'.") 

287 elif unviewable == "skip": 

288 continue 

289 data = np.zeros( 

290 len(self), dtype=np.dtype((str, key.getSize()))) 

291 for i, record in enumerate(self): 

292 data[i] = record.get(key) 

293 elif key.getTypeString() == "Flag": 

294 if not copy: 

295 if unviewable == "raise": 

296 raise ValueError("Cannot extract packed bit columns " 

297 "unless copy=True or unviewable='copy' or 'skip'.") 

298 elif unviewable == "skip": 

299 continue 

300 data = self.columns.get_bool_array(key) 

301 elif key.getTypeString() == "Angle": 

302 data = self.columns.get(key) 

303 unit = "radian" 

304 if copy: 

305 data = data.copy() 

306 elif "Array" in key.getTypeString() and key.isVariableLength(): 

307 # Can't get columns for variable-length array fields. 

308 if unviewable == "raise": 

309 raise ValueError("Cannot extract variable-length array fields unless unviewable='skip'.") 

310 elif unviewable == "skip" or unviewable == "copy": 

311 continue 

312 else: 

313 data = self.columns.get(key) 

314 if copy: 

315 data = data.copy() 

316 columns.append( 

317 astropy.table.Column( 

318 data, 

319 name=name, 

320 unit=unit, 

321 description=item.field.getDoc() 

322 ) 

323 ) 

324 return cls(columns, meta=meta, copy=False) 

325 

326 def __dir__(self): 

327 """ 

328 This custom dir is necessary due to the custom getattr below. 

329 Without it, not all of the methods available are returned with dir. 

330 See DM-7199. 

331 """ 

332 def recursive_get_class_dir(cls): 

333 """ 

334 Return a set containing the names of all methods 

335 for a given class *and* all of its subclasses. 

336 """ 

337 result = set() 

338 if cls.__bases__: 

339 for subcls in cls.__bases__: 

340 result |= recursive_get_class_dir(subcls) 

341 result |= set(cls.__dict__.keys()) 

342 return result 

343 return sorted(set(dir(self.columns)) | set(dir(self.table)) 

344 | recursive_get_class_dir(type(self)) | set(self.__dict__.keys())) 

345 

346 def __getattr__(self, name): 

347 # Catalog forwards unknown method calls to its table and column view 

348 # for convenience. (Feature requested by RHL; complaints about magic 

349 # should be directed to him.) 

350 if name == "_columns": 

351 self._columns = None 

352 return None 

353 try: 

354 return getattr(self.table, name) 

355 except AttributeError: 

356 return getattr(self.columns, name) 

357 

358 def __str__(self): 

359 if self.isContiguous(): 

360 return str(self.asAstropy()) 

361 else: 

362 fields = ' '.join(x.field.getName() for x in self.schema) 

363 return f"Non-contiguous afw.Catalog of {len(self)} rows.\ncolumns: {fields}" 

364 

365 def __repr__(self): 

366 return "%s\n%s" % (type(self), self) 

367 

368 

369Catalog.register("Base", BaseCatalog)