Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import yaml 

2import re 

3from itertools import product 

4 

5import pandas as pd 

6import numpy as np 

7import astropy.units as u 

8 

9from lsst.daf.persistence import doImport 

10from lsst.daf.butler import DeferredDatasetHandle 

11from .parquetTable import ParquetTable, MultilevelParquetTable 

12 

13 

14def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', 

15 typeKey='functor', name=None): 

16 """Initialize an object defined in a dictionary 

17 

18 The object needs to be importable as 

19 f'{basePath}.{initDict[typeKey]}' 

20 The positional and keyword arguments (if any) are contained in 

21 "args" and "kwargs" entries in the dictionary, respectively. 

22 This is used in `functors.CompositeFunctor.from_yaml` to initialize 

23 a composite functor from a specification in a YAML file. 

24 

25 Parameters 

26 ---------- 

27 initDict : dictionary 

28 Dictionary describing object's initialization. Must contain 

29 an entry keyed by ``typeKey`` that is the name of the object, 

30 relative to ``basePath``. 

31 basePath : str 

32 Path relative to module in which ``initDict[typeKey]`` is defined. 

33 typeKey : str 

34 Key of ``initDict`` that is the name of the object 

35 (relative to `basePath`). 

36 """ 

37 initDict = initDict.copy() 

38 # TO DO: DM-21956 We should be able to define functors outside this module 

39 pythonType = doImport(f'{basePath}.{initDict.pop(typeKey)}') 

40 args = [] 

41 if 'args' in initDict: 

42 args = initDict.pop('args') 

43 if isinstance(args, str): 

44 args = [args] 

45 try: 

46 element = pythonType(*args, **initDict) 

47 except Exception as e: 

48 message = f'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}' 

49 raise type(e)(message, e.args) 

50 return element 

51 

52 

53class Functor(object): 

54 """Define and execute a calculation on a ParquetTable 

55 

56 The `__call__` method accepts either a `ParquetTable` object or a 

57 `DeferredDatasetHandle`, and returns the 

58 result of the calculation as a single column. Each functor defines what 

59 columns are needed for the calculation, and only these columns are read 

60 from the `ParquetTable`. 

61 

62 The action of `__call__` consists of two steps: first, loading the 

63 necessary columns from disk into memory as a `pandas.DataFrame` object; 

64 and second, performing the computation on this dataframe and returning the 

65 result. 

66 

67 

68 To define a new `Functor`, a subclass must define a `_func` method, 

69 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`. 

70 In addition, it must define the following attributes 

71 

72 * `_columns`: The columns necessary to perform the calculation 

73 * `name`: A name appropriate for a figure axis label 

74 * `shortname`: A name appropriate for use as a dictionary key 

75 

76 On initialization, a `Functor` should declare what band (`filt` kwarg) 

77 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be 

78 applied to. This enables the `_get_data` method to extract the proper 

79 columns from the parquet file. If not specified, the dataset will fall back 

80 on the `_defaultDataset`attribute. If band is not specified and `dataset` 

81 is anything other than `'ref'`, then an error will be raised when trying to 

82 perform the calculation. 

83 

84 As currently implemented, `Functor` is only set up to expect a 

85 dataset of the format of the `deepCoadd_obj` dataset; that is, a 

86 dataframe with a multi-level column index, 

87 with the levels of the column index being `band`, 

88 `dataset`, and `column`. This is defined in the `_columnLevels` attribute, 

89 as well as being implicit in the role of the `filt` and `dataset` attributes 

90 defined at initialization. In addition, the `_get_data` method that reads 

91 the dataframe from the `ParquetTable` will return a dataframe with column 

92 index levels defined by the `_dfLevels` attribute; by default, this is 

93 `column`. 

94 

95 The `_columnLevels` and `_dfLevels` attributes should generally not need to 

96 be changed, unless `_func` needs columns from multiple filters or datasets 

97 to do the calculation. 

98 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for 

99 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe 

100 it gets to have those levels in the column index. 

101 

102 Parameters 

103 ---------- 

104 filt : str 

105 Filter upon which to do the calculation 

106 

107 dataset : str 

108 Dataset upon which to do the calculation 

109 (e.g., 'ref', 'meas', 'forced_src'). 

110 

111 """ 

112 

113 _defaultDataset = 'ref' 

114 _columnLevels = ('band', 'dataset', 'column') 

115 _dfLevels = ('column',) 

116 _defaultNoDup = False 

117 

118 def __init__(self, filt=None, dataset=None, noDup=None): 

119 self.filt = filt 

120 self.dataset = dataset if dataset is not None else self._defaultDataset 

121 self._noDup = noDup 

122 

123 @property 

124 def noDup(self): 

125 if self._noDup is not None: 

126 return self._noDup 

127 else: 

128 return self._defaultNoDup 

129 

130 @property 

131 def columns(self): 

132 """Columns required to perform calculation 

133 """ 

134 if not hasattr(self, '_columns'): 

135 raise NotImplementedError('Must define columns property or _columns attribute') 

136 return self._columns 

137 

138 def _get_data_columnLevels(self, data, columnIndex=None): 

139 """Gets the names of the column index levels 

140 

141 This should only be called in the context of a multilevel table. 

142 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable` 

143 and with the gen3 `DeferredDatasetHandle`. 

144 

145 Parameters 

146 ---------- 

147 data : `MultilevelParquetTable` or `DeferredDatasetHandle` 

148 

149 columnnIndex (optional): pandas `Index` object 

150 if not passed, then it is read from the `DeferredDatasetHandle` 

151 """ 

152 if isinstance(data, DeferredDatasetHandle): 

153 if columnIndex is None: 

154 columnIndex = data.get(component="columns") 

155 if columnIndex is not None: 

156 return columnIndex.names 

157 if isinstance(data, MultilevelParquetTable): 

158 return data.columnLevels 

159 else: 

160 raise TypeError(f"Unknown type for data: {type(data)}!") 

161 

162 def _get_data_columnLevelNames(self, data, columnIndex=None): 

163 """Gets the content of each of the column levels for a multilevel table 

164 

165 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2. 

166 

167 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable` 

168 """ 

169 if isinstance(data, DeferredDatasetHandle): 

170 if columnIndex is None: 

171 columnIndex = data.get(component="columns") 

172 if columnIndex is not None: 

173 columnLevels = columnIndex.names 

174 columnLevelNames = { 

175 level: list(np.unique(np.array([c for c in columnIndex])[:, i])) 

176 for i, level in enumerate(columnLevels) 

177 } 

178 return columnLevelNames 

179 if isinstance(data, MultilevelParquetTable): 

180 return data.columnLevelNames 

181 else: 

182 raise TypeError(f"Unknown type for data: {type(data)}!") 

183 

184 def _colsFromDict(self, colDict, columnIndex=None): 

185 """Converts dictionary column specficiation to a list of columns 

186 

187 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable` 

188 """ 

189 new_colDict = {} 

190 columnLevels = self._get_data_columnLevels(None, columnIndex=columnIndex) 

191 

192 for i, lev in enumerate(columnLevels): 

193 if lev in colDict: 

194 if isinstance(colDict[lev], str): 

195 new_colDict[lev] = [colDict[lev]] 

196 else: 

197 new_colDict[lev] = colDict[lev] 

198 else: 

199 new_colDict[lev] = columnIndex.levels[i] 

200 

201 levelCols = [new_colDict[lev] for lev in columnLevels] 

202 cols = product(*levelCols) 

203 return list(cols) 

204 

205 def multilevelColumns(self, data, columnIndex=None, returnTuple=False): 

206 """Returns columns needed by functor from multilevel dataset 

207 

208 To access tables with multilevel column structure, the `MultilevelParquetTable` 

209 or `DeferredDatasetHandle` need to be passed either a list of tuples or a 

210 dictionary. 

211 

212 Parameters 

213 ---------- 

214 data : `MultilevelParquetTable` or `DeferredDatasetHandle` 

215 

216 columnIndex (optional): pandas `Index` object 

217 either passed or read in from `DeferredDatasetHandle`. 

218 

219 `returnTuple` : bool 

220 If true, then return a list of tuples rather than the column dictionary 

221 specification. This is set to `True` by `CompositeFunctor` in order to be able to 

222 combine columns from the various component functors. 

223 

224 """ 

225 if isinstance(data, DeferredDatasetHandle) and columnIndex is None: 

226 columnIndex = data.get(component="columns") 

227 

228 # Confirm that the dataset has the column levels the functor is expecting it to have. 

229 columnLevels = self._get_data_columnLevels(data, columnIndex) 

230 

231 if not set(columnLevels) == set(self._columnLevels): 

232 raise ValueError( 

233 "ParquetTable does not have the expected column levels. " 

234 f"Got {columnLevels}; expected {self._columnLevels}." 

235 ) 

236 

237 columnDict = {'column': self.columns, 

238 'dataset': self.dataset} 

239 if self.filt is None: 

240 columnLevelNames = self._get_data_columnLevelNames(data, columnIndex) 

241 if "band" in columnLevels: 

242 if self.dataset == "ref": 

243 columnDict["band"] = columnLevelNames["band"][0] 

244 else: 

245 raise ValueError(f"'filt' not set for functor {self.name}" 

246 f"(dataset {self.dataset}) " 

247 "and ParquetTable " 

248 "contains multiple filters in column index. " 

249 "Set 'filt' or set 'dataset' to 'ref'.") 

250 else: 

251 columnDict['band'] = self.filt 

252 

253 if isinstance(data, MultilevelParquetTable): 

254 return data._colsFromDict(columnDict) 

255 elif isinstance(data, DeferredDatasetHandle): 

256 if returnTuple: 

257 return self._colsFromDict(columnDict, columnIndex=columnIndex) 

258 else: 

259 return columnDict 

260 

261 def _func(self, df, dropna=True): 

262 raise NotImplementedError('Must define calculation on dataframe') 

263 

264 def _get_columnIndex(self, data): 

265 """Return columnIndex 

266 """ 

267 

268 if isinstance(data, DeferredDatasetHandle): 

269 return data.get(component="columns") 

270 else: 

271 return None 

272 

273 def _get_data(self, data): 

274 """Retrieve dataframe necessary for calculation. 

275 

276 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle 

277 

278 Returns dataframe upon which `self._func` can act. 

279 

280 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested. 

281 """ 

282 if isinstance(data, pd.DataFrame): 

283 return data 

284 

285 # First thing to do: check to see if the data source has a multilevel column index or not. 

286 columnIndex = self._get_columnIndex(data) 

287 is_multiLevel = isinstance(data, MultilevelParquetTable) or isinstance(columnIndex, pd.MultiIndex) 

288 

289 # Simple single-level parquet table, gen2 

290 if isinstance(data, ParquetTable) and not is_multiLevel: 

291 columns = self.columns 

292 df = data.toDataFrame(columns=columns) 

293 return df 

294 

295 # Get proper multi-level columns specification for this functor 

296 if is_multiLevel: 

297 columns = self.multilevelColumns(data, columnIndex=columnIndex) 

298 

299 if isinstance(data, MultilevelParquetTable): 

300 # Load in-memory dataframe with appropriate columns the gen2 way 

301 df = data.toDataFrame(columns=columns, droplevels=False) 

302 elif isinstance(data, DeferredDatasetHandle): 

303 # Load in-memory dataframe with appropriate columns the gen3 way 

304 df = data.get(parameters={"columns": columns}) 

305 

306 # Drop unnecessary column levels 

307 df = self._setLevels(df) 

308 return df 

309 

310 def _setLevels(self, df): 

311 levelsToDrop = [n for n in df.columns.names if n not in self._dfLevels] 

312 df.columns = df.columns.droplevel(levelsToDrop) 

313 return df 

314 

315 def _dropna(self, vals): 

316 return vals.dropna() 

317 

318 def __call__(self, data, dropna=False): 

319 try: 

320 df = self._get_data(data) 

321 vals = self._func(df) 

322 except Exception: 

323 vals = self.fail(df) 

324 if dropna: 

325 vals = self._dropna(vals) 

326 

327 return vals 

328 

329 def difference(self, data1, data2, **kwargs): 

330 """Computes difference between functor called on two different ParquetTable objects 

331 """ 

332 return self(data1, **kwargs) - self(data2, **kwargs) 

333 

334 def fail(self, df): 

335 return pd.Series(np.full(len(df), np.nan), index=df.index) 

336 

337 @property 

338 def name(self): 

339 """Full name of functor (suitable for figure labels) 

340 """ 

341 return NotImplementedError 

342 

343 @property 

344 def shortname(self): 

345 """Short name of functor (suitable for column name/dict key) 

346 """ 

347 return self.name 

348 

349 

350class CompositeFunctor(Functor): 

351 """Perform multiple calculations at once on a catalog 

352 

353 The role of a `CompositeFunctor` is to group together computations from 

354 multiple functors. Instead of returning `pandas.Series` a 

355 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names 

356 being the keys of `funcDict`. 

357 

358 The `columns` attribute of a `CompositeFunctor` is the union of all columns 

359 in all the component functors. 

360 

361 A `CompositeFunctor` does not use a `_func` method itself; rather, 

362 when a `CompositeFunctor` is called, all its columns are loaded 

363 at once, and the resulting dataframe is passed to the `_func` method of each component 

364 functor. This has the advantage of only doing I/O (reading from parquet file) once, 

365 and works because each individual `_func` method of each component functor does not 

366 care if there are *extra* columns in the dataframe being passed; only that it must contain 

367 *at least* the `columns` it expects. 

368 

369 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML 

370 file specifying a collection of functors. 

371 

372 Parameters 

373 ---------- 

374 funcs : `dict` or `list` 

375 Dictionary or list of functors. If a list, then it will be converted 

376 into a dictonary according to the `.shortname` attribute of each functor. 

377 

378 """ 

379 dataset = None 

380 

381 def __init__(self, funcs, **kwargs): 

382 

383 if type(funcs) == dict: 

384 self.funcDict = funcs 

385 else: 

386 self.funcDict = {f.shortname: f for f in funcs} 

387 

388 self._filt = None 

389 

390 super().__init__(**kwargs) 

391 

392 @property 

393 def filt(self): 

394 return self._filt 

395 

396 @filt.setter 

397 def filt(self, filt): 

398 if filt is not None: 

399 for _, f in self.funcDict.items(): 

400 f.filt = filt 

401 self._filt = filt 

402 

403 def update(self, new): 

404 if isinstance(new, dict): 

405 self.funcDict.update(new) 

406 elif isinstance(new, CompositeFunctor): 

407 self.funcDict.update(new.funcDict) 

408 else: 

409 raise TypeError('Can only update with dictionary or CompositeFunctor.') 

410 

411 # Make sure new functors have the same 'filt' set 

412 if self.filt is not None: 

413 self.filt = self.filt 

414 

415 @property 

416 def columns(self): 

417 return list(set([x for y in [f.columns for f in self.funcDict.values()] for x in y])) 

418 

419 def multilevelColumns(self, data, **kwargs): 

420 # Get the union of columns for all component functors. Note the need to have `returnTuple=True` here. 

421 return list( 

422 set( 

423 [ 

424 x 

425 for y in [ 

426 f.multilevelColumns(data, returnTuple=True, **kwargs) for f in self.funcDict.values() 

427 ] 

428 for x in y 

429 ] 

430 ) 

431 ) 

432 

433 def __call__(self, data, **kwargs): 

434 """Apply the functor to the data table 

435 

436 Parameters 

437 ---------- 

438 data : `lsst.daf.butler.DeferredDatasetHandle`, 

439 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`, 

440 `lsst.pipe.tasks.parquetTable.ParquetTable`, 

441 or `pandas.DataFrame`. 

442 The table or a pointer to a table on disk from which columns can 

443 be accessed 

444 """ 

445 columnIndex = self._get_columnIndex(data) 

446 

447 # First, determine whether data has a multilevel index (either gen2 or gen3) 

448 is_multiLevel = isinstance(data, MultilevelParquetTable) or isinstance(columnIndex, pd.MultiIndex) 

449 

450 # Multilevel index, gen2 or gen3 

451 if is_multiLevel: 

452 columns = self.multilevelColumns(data, columnIndex=columnIndex) 

453 

454 if isinstance(data, MultilevelParquetTable): 

455 # Read data into memory the gen2 way 

456 df = data.toDataFrame(columns=columns, droplevels=False) 

457 elif isinstance(data, DeferredDatasetHandle): 

458 # Read data into memory the gen3 way 

459 df = data.get(parameters={"columns": columns}) 

460 

461 valDict = {} 

462 for k, f in self.funcDict.items(): 

463 try: 

464 subdf = f._setLevels( 

465 df[f.multilevelColumns(data, returnTuple=True, columnIndex=columnIndex)] 

466 ) 

467 valDict[k] = f._func(subdf) 

468 except Exception: 

469 valDict[k] = f.fail(subdf) 

470 

471 else: 

472 if isinstance(data, DeferredDatasetHandle): 

473 # input if Gen3 deferLoad=True 

474 df = data.get(columns=self.columns) 

475 elif isinstance(data, pd.DataFrame): 

476 # input if Gen3 deferLoad=False 

477 df = data 

478 else: 

479 # Original Gen2 input is type ParquetTable and the fallback 

480 df = data.toDataFrame(columns=self.columns) 

481 

482 valDict = {k: f._func(df) for k, f in self.funcDict.items()} 

483 

484 try: 

485 valDf = pd.concat(valDict, axis=1) 

486 except TypeError: 

487 print([(k, type(v)) for k, v in valDict.items()]) 

488 raise 

489 

490 if kwargs.get('dropna', False): 

491 valDf = valDf.dropna(how='any') 

492 

493 return valDf 

494 

495 @classmethod 

496 def renameCol(cls, col, renameRules): 

497 if renameRules is None: 

498 return col 

499 for old, new in renameRules: 

500 if col.startswith(old): 

501 col = col.replace(old, new) 

502 return col 

503 

504 @classmethod 

505 def from_file(cls, filename, **kwargs): 

506 with open(filename) as f: 

507 translationDefinition = yaml.safe_load(f) 

508 

509 return cls.from_yaml(translationDefinition, **kwargs) 

510 

511 @classmethod 

512 def from_yaml(cls, translationDefinition, **kwargs): 

513 funcs = {} 

514 for func, val in translationDefinition['funcs'].items(): 

515 funcs[func] = init_fromDict(val, name=func) 

516 

517 if 'flag_rename_rules' in translationDefinition: 

518 renameRules = translationDefinition['flag_rename_rules'] 

519 else: 

520 renameRules = None 

521 

522 if 'refFlags' in translationDefinition: 

523 for flag in translationDefinition['refFlags']: 

524 funcs[cls.renameCol(flag, renameRules)] = Column(flag, dataset='ref') 

525 

526 if 'flags' in translationDefinition: 

527 for flag in translationDefinition['flags']: 

528 funcs[cls.renameCol(flag, renameRules)] = Column(flag, dataset='meas') 

529 

530 return cls(funcs, **kwargs) 

531 

532 

533def mag_aware_eval(df, expr): 

534 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means 

535 

536 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes. 

537 

538 Parameters 

539 ---------- 

540 df : pandas.DataFrame 

541 Dataframe on which to evaluate expression. 

542 

543 expr : str 

544 Expression. 

545 """ 

546 try: 

547 expr_new = re.sub(r'mag\((\w+)\)', r'-2.5*log(\g<1>)/log(10)', expr) 

548 val = df.eval(expr_new, truediv=True) 

549 except Exception: # Should check what actually gets raised 

550 expr_new = re.sub(r'mag\((\w+)\)', r'-2.5*log(\g<1>_instFlux)/log(10)', expr) 

551 val = df.eval(expr_new, truediv=True) 

552 return val 

553 

554 

555class CustomFunctor(Functor): 

556 """Arbitrary computation on a catalog 

557 

558 Column names (and thus the columns to be loaded from catalog) are found 

559 by finding all words and trying to ignore all "math-y" words. 

560 

561 Parameters 

562 ---------- 

563 expr : str 

564 Expression to evaluate, to be parsed and executed by `mag_aware_eval`. 

565 """ 

566 _ignore_words = ('mag', 'sin', 'cos', 'exp', 'log', 'sqrt') 

567 

568 def __init__(self, expr, **kwargs): 

569 self.expr = expr 

570 super().__init__(**kwargs) 

571 

572 @property 

573 def name(self): 

574 return self.expr 

575 

576 @property 

577 def columns(self): 

578 flux_cols = re.findall(r'mag\(\s*(\w+)\s*\)', self.expr) 

579 

580 cols = [c for c in re.findall(r'[a-zA-Z_]+', self.expr) if c not in self._ignore_words] 

581 not_a_col = [] 

582 for c in flux_cols: 

583 if not re.search('_instFlux$', c): 

584 cols.append(f'{c}_instFlux') 

585 not_a_col.append(c) 

586 else: 

587 cols.append(c) 

588 

589 return list(set([c for c in cols if c not in not_a_col])) 

590 

591 def _func(self, df): 

592 return mag_aware_eval(df, self.expr) 

593 

594 

595class Column(Functor): 

596 """Get column with specified name 

597 """ 

598 

599 def __init__(self, col, **kwargs): 

600 self.col = col 

601 super().__init__(**kwargs) 

602 

603 @property 

604 def name(self): 

605 return self.col 

606 

607 @property 

608 def columns(self): 

609 return [self.col] 

610 

611 def _func(self, df): 

612 return df[self.col] 

613 

614 

615class Index(Functor): 

616 """Return the value of the index for each object 

617 """ 

618 

619 columns = ['coord_ra'] # just a dummy; something has to be here 

620 _defaultDataset = 'ref' 

621 _defaultNoDup = True 

622 

623 def _func(self, df): 

624 return pd.Series(df.index, index=df.index) 

625 

626 

627class IDColumn(Column): 

628 col = 'id' 

629 _allow_difference = False 

630 _defaultNoDup = True 

631 

632 def _func(self, df): 

633 return pd.Series(df.index, index=df.index) 

634 

635 

636class FootprintNPix(Column): 

637 col = 'base_Footprint_nPix' 

638 

639 

640class CoordColumn(Column): 

641 """Base class for coordinate column, in degrees 

642 """ 

643 _radians = True 

644 

645 def __init__(self, col, **kwargs): 

646 super().__init__(col, **kwargs) 

647 

648 def _func(self, df): 

649 # Must not modify original column in case that column is used by another functor 

650 output = df[self.col] * 180 / np.pi if self._radians else df[self.col] 

651 return output 

652 

653 

654class RAColumn(CoordColumn): 

655 """Right Ascension, in degrees 

656 """ 

657 name = 'RA' 

658 _defaultNoDup = True 

659 

660 def __init__(self, **kwargs): 

661 super().__init__('coord_ra', **kwargs) 

662 

663 def __call__(self, catalog, **kwargs): 

664 return super().__call__(catalog, **kwargs) 

665 

666 

667class DecColumn(CoordColumn): 

668 """Declination, in degrees 

669 """ 

670 name = 'Dec' 

671 _defaultNoDup = True 

672 

673 def __init__(self, **kwargs): 

674 super().__init__('coord_dec', **kwargs) 

675 

676 def __call__(self, catalog, **kwargs): 

677 return super().__call__(catalog, **kwargs) 

678 

679 

680def fluxName(col): 

681 if not col.endswith('_instFlux'): 

682 col += '_instFlux' 

683 return col 

684 

685 

686def fluxErrName(col): 

687 if not col.endswith('_instFluxErr'): 

688 col += '_instFluxErr' 

689 return col 

690 

691 

692class Mag(Functor): 

693 """Compute calibrated magnitude 

694 

695 Takes a `calib` argument, which returns the flux at mag=0 

696 as `calib.getFluxMag0()`. If not provided, then the default 

697 `fluxMag0` is 63095734448.0194, which is default for HSC. 

698 This default should be removed in DM-21955 

699 

700 This calculation hides warnings about invalid values and dividing by zero. 

701 

702 As for all functors, a `dataset` and `filt` kwarg should be provided upon 

703 initialization. Unlike the default `Functor`, however, the default dataset 

704 for a `Mag` is `'meas'`, rather than `'ref'`. 

705 

706 Parameters 

707 ---------- 

708 col : `str` 

709 Name of flux column from which to compute magnitude. Can be parseable 

710 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass 

711 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will 

712 understand. 

713 calib : `lsst.afw.image.calib.Calib` (optional) 

714 Object that knows zero point. 

715 """ 

716 _defaultDataset = 'meas' 

717 

718 def __init__(self, col, calib=None, **kwargs): 

719 self.col = fluxName(col) 

720 self.calib = calib 

721 if calib is not None: 

722 self.fluxMag0 = calib.getFluxMag0()[0] 

723 else: 

724 # TO DO: DM-21955 Replace hard coded photometic calibration values 

725 self.fluxMag0 = 63095734448.0194 

726 

727 super().__init__(**kwargs) 

728 

729 @property 

730 def columns(self): 

731 return [self.col] 

732 

733 def _func(self, df): 

734 with np.warnings.catch_warnings(): 

735 np.warnings.filterwarnings('ignore', r'invalid value encountered') 

736 np.warnings.filterwarnings('ignore', r'divide by zero') 

737 return -2.5*np.log10(df[self.col] / self.fluxMag0) 

738 

739 @property 

740 def name(self): 

741 return f'mag_{self.col}' 

742 

743 

744class MagErr(Mag): 

745 """Compute calibrated magnitude uncertainty 

746 

747 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`. 

748 

749 Parameters 

750 col : `str` 

751 Name of flux column 

752 calib : `lsst.afw.image.calib.Calib` (optional) 

753 Object that knows zero point. 

754 """ 

755 

756 def __init__(self, *args, **kwargs): 

757 super().__init__(*args, **kwargs) 

758 if self.calib is not None: 

759 self.fluxMag0Err = self.calib.getFluxMag0()[1] 

760 else: 

761 self.fluxMag0Err = 0. 

762 

763 @property 

764 def columns(self): 

765 return [self.col, self.col + 'Err'] 

766 

767 def _func(self, df): 

768 with np.warnings.catch_warnings(): 

769 np.warnings.filterwarnings('ignore', r'invalid value encountered') 

770 np.warnings.filterwarnings('ignore', r'divide by zero') 

771 fluxCol, fluxErrCol = self.columns 

772 x = df[fluxErrCol] / df[fluxCol] 

773 y = self.fluxMag0Err / self.fluxMag0 

774 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y) 

775 return magErr 

776 

777 @property 

778 def name(self): 

779 return super().name + '_err' 

780 

781 

782class NanoMaggie(Mag): 

783 """ 

784 """ 

785 

786 def _func(self, df): 

787 return (df[self.col] / self.fluxMag0) * 1e9 

788 

789 

790class MagDiff(Functor): 

791 _defaultDataset = 'meas' 

792 

793 """Functor to calculate magnitude difference""" 

794 

795 def __init__(self, col1, col2, **kwargs): 

796 self.col1 = fluxName(col1) 

797 self.col2 = fluxName(col2) 

798 super().__init__(**kwargs) 

799 

800 @property 

801 def columns(self): 

802 return [self.col1, self.col2] 

803 

804 def _func(self, df): 

805 with np.warnings.catch_warnings(): 

806 np.warnings.filterwarnings('ignore', r'invalid value encountered') 

807 np.warnings.filterwarnings('ignore', r'divide by zero') 

808 return -2.5*np.log10(df[self.col1]/df[self.col2]) 

809 

810 @property 

811 def name(self): 

812 return f'(mag_{self.col1} - mag_{self.col2})' 

813 

814 @property 

815 def shortname(self): 

816 return f'magDiff_{self.col1}_{self.col2}' 

817 

818 

819class Color(Functor): 

820 """Compute the color between two filters 

821 

822 Computes color by initializing two different `Mag` 

823 functors based on the `col` and filters provided, and 

824 then returning the difference. 

825 

826 This is enabled by the `_func` expecting a dataframe with a 

827 multilevel column index, with both `'band'` and `'column'`, 

828 instead of just `'column'`, which is the `Functor` default. 

829 This is controlled by the `_dfLevels` attribute. 

830 

831 Also of note, the default dataset for `Color` is `forced_src'`, 

832 whereas for `Mag` it is `'meas'`. 

833 

834 Parameters 

835 ---------- 

836 col : str 

837 Name of flux column from which to compute; same as would be passed to 

838 `lsst.pipe.tasks.functors.Mag`. 

839 

840 filt2, filt1 : str 

841 Filters from which to compute magnitude difference. 

842 Color computed is `Mag(filt2) - Mag(filt1)`. 

843 """ 

844 _defaultDataset = 'forced_src' 

845 _dfLevels = ('band', 'column') 

846 _defaultNoDup = True 

847 

848 def __init__(self, col, filt2, filt1, **kwargs): 

849 self.col = fluxName(col) 

850 if filt2 == filt1: 

851 raise RuntimeError("Cannot compute Color for %s: %s - %s " % (col, filt2, filt1)) 

852 self.filt2 = filt2 

853 self.filt1 = filt1 

854 

855 self.mag2 = Mag(col, filt=filt2, **kwargs) 

856 self.mag1 = Mag(col, filt=filt1, **kwargs) 

857 

858 super().__init__(**kwargs) 

859 

860 @property 

861 def filt(self): 

862 return None 

863 

864 @filt.setter 

865 def filt(self, filt): 

866 pass 

867 

868 def _func(self, df): 

869 mag2 = self.mag2._func(df[self.filt2]) 

870 mag1 = self.mag1._func(df[self.filt1]) 

871 return mag2 - mag1 

872 

873 @property 

874 def columns(self): 

875 return [self.mag1.col, self.mag2.col] 

876 

877 def multilevelColumns(self, parq, **kwargs): 

878 return [(self.dataset, self.filt1, self.col), (self.dataset, self.filt2, self.col)] 

879 

880 @property 

881 def name(self): 

882 return f'{self.filt2} - {self.filt1} ({self.col})' 

883 

884 @property 

885 def shortname(self): 

886 return f"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}" 

887 

888 

889class Labeller(Functor): 

890 """Main function of this subclass is to override the dropna=True 

891 """ 

892 _null_label = 'null' 

893 _allow_difference = False 

894 name = 'label' 

895 _force_str = False 

896 

897 def __call__(self, parq, dropna=False, **kwargs): 

898 return super().__call__(parq, dropna=False, **kwargs) 

899 

900 

901class StarGalaxyLabeller(Labeller): 

902 _columns = ["base_ClassificationExtendedness_value"] 

903 _column = "base_ClassificationExtendedness_value" 

904 

905 def _func(self, df): 

906 x = df[self._columns][self._column] 

907 mask = x.isnull() 

908 test = (x < 0.5).astype(int) 

909 test = test.mask(mask, 2) 

910 

911 # TODO: DM-21954 Look into veracity of inline comment below 

912 # are these backwards? 

913 categories = ['galaxy', 'star', self._null_label] 

914 label = pd.Series(pd.Categorical.from_codes(test, categories=categories), 

915 index=x.index, name='label') 

916 if self._force_str: 

917 label = label.astype(str) 

918 return label 

919 

920 

921class NumStarLabeller(Labeller): 

922 _columns = ['numStarFlags'] 

923 labels = {"star": 0, "maybe": 1, "notStar": 2} 

924 

925 def _func(self, df): 

926 x = df[self._columns][self._columns[0]] 

927 

928 # Number of filters 

929 n = len(x.unique()) - 1 

930 

931 labels = ['noStar', 'maybe', 'star'] 

932 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels), 

933 index=x.index, name='label') 

934 

935 if self._force_str: 

936 label = label.astype(str) 

937 

938 return label 

939 

940 

941class DeconvolvedMoments(Functor): 

942 name = 'Deconvolved Moments' 

943 shortname = 'deconvolvedMoments' 

944 _columns = ("ext_shapeHSM_HsmSourceMoments_xx", 

945 "ext_shapeHSM_HsmSourceMoments_yy", 

946 "base_SdssShape_xx", "base_SdssShape_yy", 

947 "ext_shapeHSM_HsmPsfMoments_xx", 

948 "ext_shapeHSM_HsmPsfMoments_yy") 

949 

950 def _func(self, df): 

951 """Calculate deconvolved moments""" 

952 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns: # _xx added by tdm 

953 hsm = df["ext_shapeHSM_HsmSourceMoments_xx"] + df["ext_shapeHSM_HsmSourceMoments_yy"] 

954 else: 

955 hsm = np.ones(len(df))*np.nan 

956 sdss = df["base_SdssShape_xx"] + df["base_SdssShape_yy"] 

957 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns: 

958 psf = df["ext_shapeHSM_HsmPsfMoments_xx"] + df["ext_shapeHSM_HsmPsfMoments_yy"] 

959 else: 

960 # LSST does not have shape.sdss.psf. Could instead add base_PsfShape to catalog using 

961 # exposure.getPsf().computeShape(s.getCentroid()).getIxx() 

962 # raise TaskError("No psf shape parameter found in catalog") 

963 raise RuntimeError('No psf shape parameter found in catalog') 

964 

965 return hsm.where(np.isfinite(hsm), sdss) - psf 

966 

967 

968class SdssTraceSize(Functor): 

969 """Functor to calculate SDSS trace radius size for sources""" 

970 name = "SDSS Trace Size" 

971 shortname = 'sdssTrace' 

972 _columns = ("base_SdssShape_xx", "base_SdssShape_yy") 

973 

974 def _func(self, df): 

975 srcSize = np.sqrt(0.5*(df["base_SdssShape_xx"] + df["base_SdssShape_yy"])) 

976 return srcSize 

977 

978 

979class PsfSdssTraceSizeDiff(Functor): 

980 """Functor to calculate SDSS trace radius size difference (%) between object and psf model""" 

981 name = "PSF - SDSS Trace Size" 

982 shortname = 'psf_sdssTrace' 

983 _columns = ("base_SdssShape_xx", "base_SdssShape_yy", 

984 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy") 

985 

986 def _func(self, df): 

987 srcSize = np.sqrt(0.5*(df["base_SdssShape_xx"] + df["base_SdssShape_yy"])) 

988 psfSize = np.sqrt(0.5*(df["base_SdssShape_psf_xx"] + df["base_SdssShape_psf_yy"])) 

989 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize)) 

990 return sizeDiff 

991 

992 

993class HsmTraceSize(Functor): 

994 """Functor to calculate HSM trace radius size for sources""" 

995 name = 'HSM Trace Size' 

996 shortname = 'hsmTrace' 

997 _columns = ("ext_shapeHSM_HsmSourceMoments_xx", 

998 "ext_shapeHSM_HsmSourceMoments_yy") 

999 

1000 def _func(self, df): 

1001 srcSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmSourceMoments_xx"] 

1002 + df["ext_shapeHSM_HsmSourceMoments_yy"])) 

1003 return srcSize 

1004 

1005 

1006class PsfHsmTraceSizeDiff(Functor): 

1007 """Functor to calculate HSM trace radius size difference (%) between object and psf model""" 

1008 name = 'PSF - HSM Trace Size' 

1009 shortname = 'psf_HsmTrace' 

1010 _columns = ("ext_shapeHSM_HsmSourceMoments_xx", 

1011 "ext_shapeHSM_HsmSourceMoments_yy", 

1012 "ext_shapeHSM_HsmPsfMoments_xx", 

1013 "ext_shapeHSM_HsmPsfMoments_yy") 

1014 

1015 def _func(self, df): 

1016 srcSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmSourceMoments_xx"] 

1017 + df["ext_shapeHSM_HsmSourceMoments_yy"])) 

1018 psfSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmPsfMoments_xx"] 

1019 + df["ext_shapeHSM_HsmPsfMoments_yy"])) 

1020 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize)) 

1021 return sizeDiff 

1022 

1023 

1024class HsmFwhm(Functor): 

1025 name = 'HSM Psf FWHM' 

1026 _columns = ('ext_shapeHSM_HsmPsfMoments_xx', 'ext_shapeHSM_HsmPsfMoments_yy') 

1027 # TODO: DM-21403 pixel scale should be computed from the CD matrix or transform matrix 

1028 pixelScale = 0.168 

1029 SIGMA2FWHM = 2*np.sqrt(2*np.log(2)) 

1030 

1031 def _func(self, df): 

1032 return self.pixelScale*self.SIGMA2FWHM*np.sqrt( 

1033 0.5*(df['ext_shapeHSM_HsmPsfMoments_xx'] + df['ext_shapeHSM_HsmPsfMoments_yy'])) 

1034 

1035 

1036class E1(Functor): 

1037 name = "Distortion Ellipticity (e1)" 

1038 shortname = "Distortion" 

1039 

1040 def __init__(self, colXX, colXY, colYY, **kwargs): 

1041 self.colXX = colXX 

1042 self.colXY = colXY 

1043 self.colYY = colYY 

1044 self._columns = [self.colXX, self.colXY, self.colYY] 

1045 super().__init__(**kwargs) 

1046 

1047 @property 

1048 def columns(self): 

1049 return [self.colXX, self.colXY, self.colYY] 

1050 

1051 def _func(self, df): 

1052 return df[self.colXX] - df[self.colYY] / (df[self.colXX] + df[self.colYY]) 

1053 

1054 

1055class E2(Functor): 

1056 name = "Ellipticity e2" 

1057 

1058 def __init__(self, colXX, colXY, colYY, **kwargs): 

1059 self.colXX = colXX 

1060 self.colXY = colXY 

1061 self.colYY = colYY 

1062 super().__init__(**kwargs) 

1063 

1064 @property 

1065 def columns(self): 

1066 return [self.colXX, self.colXY, self.colYY] 

1067 

1068 def _func(self, df): 

1069 return 2*df[self.colXY] / (df[self.colXX] + df[self.colYY]) 

1070 

1071 

1072class RadiusFromQuadrupole(Functor): 

1073 

1074 def __init__(self, colXX, colXY, colYY, **kwargs): 

1075 self.colXX = colXX 

1076 self.colXY = colXY 

1077 self.colYY = colYY 

1078 super().__init__(**kwargs) 

1079 

1080 @property 

1081 def columns(self): 

1082 return [self.colXX, self.colXY, self.colYY] 

1083 

1084 def _func(self, df): 

1085 return (df[self.colXX]*df[self.colYY] - df[self.colXY]**2)**0.25 

1086 

1087 

1088class LocalWcs(Functor): 

1089 """Computations using the stored localWcs. 

1090 """ 

1091 name = "LocalWcsOperations" 

1092 

1093 def __init__(self, 

1094 colCD_1_1, 

1095 colCD_1_2, 

1096 colCD_2_1, 

1097 colCD_2_2, 

1098 **kwargs): 

1099 self.colCD_1_1 = colCD_1_1 

1100 self.colCD_1_2 = colCD_1_2 

1101 self.colCD_2_1 = colCD_2_1 

1102 self.colCD_2_2 = colCD_2_2 

1103 super().__init__(**kwargs) 

1104 

1105 def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22): 

1106 """Compute the distance on the sphere from x2, y1 to x1, y1. 

1107 

1108 Parameters 

1109 ---------- 

1110 x : `pandas.Series` 

1111 X pixel coordinate. 

1112 y : `pandas.Series` 

1113 Y pixel coordinate. 

1114 cd11 : `pandas.Series` 

1115 [1, 1] element of the local Wcs affine transform. 

1116 cd11 : `pandas.Series` 

1117 [1, 1] element of the local Wcs affine transform. 

1118 cd12 : `pandas.Series` 

1119 [1, 2] element of the local Wcs affine transform. 

1120 cd21 : `pandas.Series` 

1121 [2, 1] element of the local Wcs affine transform. 

1122 cd22 : `pandas.Series` 

1123 [2, 2] element of the local Wcs affine transform. 

1124 

1125 Returns 

1126 ------- 

1127 raDecTuple : tuple 

1128 RA and dec conversion of x and y given the local Wcs. Returned 

1129 units are in radians. 

1130 

1131 """ 

1132 return (x * cd11 + y * cd12, x * cd21 + y * cd22) 

1133 

1134 def computeSkySeperation(self, ra1, dec1, ra2, dec2): 

1135 """Compute the local pixel scale conversion. 

1136 

1137 Parameters 

1138 ---------- 

1139 ra1 : `pandas.Series` 

1140 Ra of the first coordinate in radians. 

1141 dec1 : `pandas.Series` 

1142 Dec of the first coordinate in radians. 

1143 ra2 : `pandas.Series` 

1144 Ra of the second coordinate in radians. 

1145 dec2 : `pandas.Series` 

1146 Dec of the second coordinate in radians. 

1147 

1148 Returns 

1149 ------- 

1150 dist : `pandas.Series` 

1151 Distance on the sphere in radians. 

1152 """ 

1153 deltaDec = dec2 - dec1 

1154 deltaRa = ra2 - ra1 

1155 return 2 * np.arcsin( 

1156 np.sqrt( 

1157 np.sin(deltaDec / 2) ** 2 

1158 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2)) 

1159 

1160 def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22): 

1161 """Compute the distance on the sphere from x2, y1 to x1, y1. 

1162 

1163 Parameters 

1164 ---------- 

1165 x1 : `pandas.Series` 

1166 X pixel coordinate. 

1167 y1 : `pandas.Series` 

1168 Y pixel coordinate. 

1169 x2 : `pandas.Series` 

1170 X pixel coordinate. 

1171 y2 : `pandas.Series` 

1172 Y pixel coordinate. 

1173 cd11 : `pandas.Series` 

1174 [1, 1] element of the local Wcs affine transform. 

1175 cd11 : `pandas.Series` 

1176 [1, 1] element of the local Wcs affine transform. 

1177 cd12 : `pandas.Series` 

1178 [1, 2] element of the local Wcs affine transform. 

1179 cd21 : `pandas.Series` 

1180 [2, 1] element of the local Wcs affine transform. 

1181 cd22 : `pandas.Series` 

1182 [2, 2] element of the local Wcs affine transform. 

1183 

1184 Returns 

1185 ------- 

1186 Distance : `pandas.Series` 

1187 Arcseconds per pixel at the location of the local WC 

1188 """ 

1189 ra1, dec1 = self.computeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22) 

1190 ra2, dec2 = self.computeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22) 

1191 # Great circle distance for small separations. 

1192 return self.computeSkySeperation(ra1, dec1, ra2, dec2) 

1193 

1194 

1195class ComputePixelScale(LocalWcs): 

1196 """Compute the local pixel scale from the stored CDMatrix. 

1197 """ 

1198 name = "PixelScale" 

1199 

1200 @property 

1201 def columns(self): 

1202 return [self.colCD_1_1, 

1203 self.colCD_1_2, 

1204 self.colCD_2_1, 

1205 self.colCD_2_2] 

1206 

1207 def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22): 

1208 """Compute the local pixel to scale conversion in arcseconds. 

1209 

1210 Parameters 

1211 ---------- 

1212 cd11 : `pandas.Series` 

1213 [1, 1] element of the local Wcs affine transform in radians. 

1214 cd11 : `pandas.Series` 

1215 [1, 1] element of the local Wcs affine transform in radians. 

1216 cd12 : `pandas.Series` 

1217 [1, 2] element of the local Wcs affine transform in radians. 

1218 cd21 : `pandas.Series` 

1219 [2, 1] element of the local Wcs affine transform in radians. 

1220 cd22 : `pandas.Series` 

1221 [2, 2] element of the local Wcs affine transform in radians. 

1222 

1223 Returns 

1224 ------- 

1225 pixScale : `pandas.Series` 

1226 Arcseconds per pixel at the location of the local WC 

1227 """ 

1228 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21))) 

1229 

1230 def _func(self, df): 

1231 return self.pixelScaleArcseconds(df[self.colCD_1_1], 

1232 df[self.colCD_1_2], 

1233 df[self.colCD_2_1], 

1234 df[self.colCD_2_2]) 

1235 

1236 

1237class ConvertPixelToArcseconds(ComputePixelScale): 

1238 """Convert a value in units pixels to units arcseconds. 

1239 """ 

1240 

1241 def __init__(self, 

1242 col, 

1243 colCD_1_1, 

1244 colCD_1_2, 

1245 colCD_2_1, 

1246 colCD_2_2, 

1247 **kwargs): 

1248 self.col = col 

1249 super().__init__(colCD_1_1, 

1250 colCD_1_2, 

1251 colCD_2_1, 

1252 colCD_2_2, 

1253 **kwargs) 

1254 

1255 @property 

1256 def name(self): 

1257 return f"{self.col}_asArcseconds" 

1258 

1259 @property 

1260 def columns(self): 

1261 return [self.col, 

1262 self.colCD_1_1, 

1263 self.colCD_1_2, 

1264 self.colCD_2_1, 

1265 self.colCD_2_2] 

1266 

1267 def _func(self, df): 

1268 return df[self.col] * self.pixelScaleArcseconds(df[self.colCD_1_1], 

1269 df[self.colCD_1_2], 

1270 df[self.colCD_2_1], 

1271 df[self.colCD_2_2]) 

1272 

1273 

1274class ReferenceBand(Functor): 

1275 name = 'Reference Band' 

1276 shortname = 'refBand' 

1277 

1278 @property 

1279 def columns(self): 

1280 return ["merge_measurement_i", 

1281 "merge_measurement_r", 

1282 "merge_measurement_z", 

1283 "merge_measurement_y", 

1284 "merge_measurement_g"] 

1285 

1286 def _func(self, df): 

1287 def getFilterAliasName(row): 

1288 # get column name with the max value (True > False) 

1289 colName = row.idxmax() 

1290 return colName.replace('merge_measurement_', '') 

1291 

1292 return df[self.columns].apply(getFilterAliasName, axis=1) 

1293 

1294 

1295class Photometry(Functor): 

1296 # AB to NanoJansky (3631 Jansky) 

1297 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy) 

1298 LOG_AB_FLUX_SCALE = 12.56 

1299 FIVE_OVER_2LOG10 = 1.085736204758129569 

1300 # TO DO: DM-21955 Replace hard coded photometic calibration values 

1301 COADD_ZP = 27 

1302 

1303 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs): 

1304 self.vhypot = np.vectorize(self.hypot) 

1305 self.col = colFlux 

1306 self.colFluxErr = colFluxErr 

1307 

1308 self.calib = calib 

1309 if calib is not None: 

1310 self.fluxMag0, self.fluxMag0Err = calib.getFluxMag0() 

1311 else: 

1312 self.fluxMag0 = 1./np.power(10, -0.4*self.COADD_ZP) 

1313 self.fluxMag0Err = 0. 

1314 

1315 super().__init__(**kwargs) 

1316 

1317 @property 

1318 def columns(self): 

1319 return [self.col] 

1320 

1321 @property 

1322 def name(self): 

1323 return f'mag_{self.col}' 

1324 

1325 @classmethod 

1326 def hypot(cls, a, b): 

1327 if np.abs(a) < np.abs(b): 

1328 a, b = b, a 

1329 if a == 0.: 

1330 return 0. 

1331 q = b/a 

1332 return np.abs(a) * np.sqrt(1. + q*q) 

1333 

1334 def dn2flux(self, dn, fluxMag0): 

1335 return self.AB_FLUX_SCALE * dn / fluxMag0 

1336 

1337 def dn2mag(self, dn, fluxMag0): 

1338 with np.warnings.catch_warnings(): 

1339 np.warnings.filterwarnings('ignore', r'invalid value encountered') 

1340 np.warnings.filterwarnings('ignore', r'divide by zero') 

1341 return -2.5 * np.log10(dn/fluxMag0) 

1342 

1343 def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err): 

1344 retVal = self.vhypot(dn * fluxMag0Err, dnErr * fluxMag0) 

1345 retVal *= self.AB_FLUX_SCALE / fluxMag0 / fluxMag0 

1346 return retVal 

1347 

1348 def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err): 

1349 retVal = self.dn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.dn2flux(dn, fluxMag0) 

1350 return self.FIVE_OVER_2LOG10 * retVal 

1351 

1352 

1353class NanoJansky(Photometry): 

1354 def _func(self, df): 

1355 return self.dn2flux(df[self.col], self.fluxMag0) 

1356 

1357 

1358class NanoJanskyErr(Photometry): 

1359 @property 

1360 def columns(self): 

1361 return [self.col, self.colFluxErr] 

1362 

1363 def _func(self, df): 

1364 retArr = self.dn2fluxErr(df[self.col], df[self.colFluxErr], self.fluxMag0, self.fluxMag0Err) 

1365 return pd.Series(retArr, index=df.index) 

1366 

1367 

1368class Magnitude(Photometry): 

1369 def _func(self, df): 

1370 return self.dn2mag(df[self.col], self.fluxMag0) 

1371 

1372 

1373class MagnitudeErr(Photometry): 

1374 @property 

1375 def columns(self): 

1376 return [self.col, self.colFluxErr] 

1377 

1378 def _func(self, df): 

1379 retArr = self.dn2MagErr(df[self.col], df[self.colFluxErr], self.fluxMag0, self.fluxMag0Err) 

1380 return pd.Series(retArr, index=df.index) 

1381 

1382 

1383class LocalPhotometry(Functor): 

1384 """Base class for calibrating the specified instrument flux column using 

1385 the local photometric calibration. 

1386 

1387 Parameters 

1388 ---------- 

1389 instFluxCol : `str` 

1390 Name of the instrument flux column. 

1391 instFluxErrCol : `str` 

1392 Name of the assocated error columns for ``instFluxCol``. 

1393 photoCalibCol : `str` 

1394 Name of local calibration column. 

1395 photoCalibErrCol : `str` 

1396 Error associated with ``photoCalibCol`` 

1397 

1398 See also 

1399 -------- 

1400 LocalPhotometry 

1401 LocalNanojansky 

1402 LocalNanojanskyErr 

1403 LocalMagnitude 

1404 LocalMagnitudeErr 

1405 """ 

1406 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag) 

1407 

1408 def __init__(self, 

1409 instFluxCol, 

1410 instFluxErrCol, 

1411 photoCalibCol, 

1412 photoCalibErrCol, 

1413 **kwargs): 

1414 self.instFluxCol = instFluxCol 

1415 self.instFluxErrCol = instFluxErrCol 

1416 self.photoCalibCol = photoCalibCol 

1417 self.photoCalibErrCol = photoCalibErrCol 

1418 super().__init__(**kwargs) 

1419 

1420 def instFluxToNanojansky(self, instFlux, localCalib): 

1421 """Convert instrument flux to nanojanskys. 

1422 

1423 Parameters 

1424 ---------- 

1425 instFlux : `numpy.ndarray` or `pandas.Series` 

1426 Array of instrument flux measurements 

1427 localCalib : `numpy.ndarray` or `pandas.Series` 

1428 Array of local photometric calibration estimates. 

1429 

1430 Returns 

1431 ------- 

1432 calibFlux : `numpy.ndarray` or `pandas.Series` 

1433 Array of calibrated flux measurements. 

1434 """ 

1435 return instFlux * localCalib 

1436 

1437 def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr): 

1438 """Convert instrument flux to nanojanskys. 

1439 

1440 Parameters 

1441 ---------- 

1442 instFlux : `numpy.ndarray` or `pandas.Series` 

1443 Array of instrument flux measurements 

1444 instFluxErr : `numpy.ndarray` or `pandas.Series` 

1445 Errors on associated ``instFlux`` values 

1446 localCalib : `numpy.ndarray` or `pandas.Series` 

1447 Array of local photometric calibration estimates. 

1448 localCalibErr : `numpy.ndarray` or `pandas.Series` 

1449 Errors on associated ``localCalib`` values 

1450 

1451 Returns 

1452 ------- 

1453 calibFluxErr : `numpy.ndarray` or `pandas.Series` 

1454 Errors on calibrated flux measurements. 

1455 """ 

1456 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr) 

1457 

1458 def instFluxToMagnitude(self, instFlux, localCalib): 

1459 """Convert instrument flux to nanojanskys. 

1460 

1461 Parameters 

1462 ---------- 

1463 instFlux : `numpy.ndarray` or `pandas.Series` 

1464 Array of instrument flux measurements 

1465 localCalib : `numpy.ndarray` or `pandas.Series` 

1466 Array of local photometric calibration estimates. 

1467 

1468 Returns 

1469 ------- 

1470 calibMag : `numpy.ndarray` or `pandas.Series` 

1471 Array of calibrated AB magnitudes. 

1472 """ 

1473 return -2.5 * np.log10(self.instFluxToNanojansky(instFlux, localCalib)) + self.logNJanskyToAB 

1474 

1475 def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr): 

1476 """Convert instrument flux err to nanojanskys. 

1477 

1478 Parameters 

1479 ---------- 

1480 instFlux : `numpy.ndarray` or `pandas.Series` 

1481 Array of instrument flux measurements 

1482 instFluxErr : `numpy.ndarray` or `pandas.Series` 

1483 Errors on associated ``instFlux`` values 

1484 localCalib : `numpy.ndarray` or `pandas.Series` 

1485 Array of local photometric calibration estimates. 

1486 localCalibErr : `numpy.ndarray` or `pandas.Series` 

1487 Errors on associated ``localCalib`` values 

1488 

1489 Returns 

1490 ------- 

1491 calibMagErr: `numpy.ndarray` or `pandas.Series` 

1492 Error on calibrated AB magnitudes. 

1493 """ 

1494 err = self.instFluxErrToNanojanskyErr(instFlux, instFluxErr, localCalib, localCalibErr) 

1495 return 2.5 / np.log(10) * err / self.instFluxToNanojansky(instFlux, instFluxErr) 

1496 

1497 

1498class LocalNanojansky(LocalPhotometry): 

1499 """Compute calibrated fluxes using the local calibration value. 

1500 

1501 See also 

1502 -------- 

1503 LocalNanojansky 

1504 LocalNanojanskyErr 

1505 LocalMagnitude 

1506 LocalMagnitudeErr 

1507 """ 

1508 

1509 @property 

1510 def columns(self): 

1511 return [self.instFluxCol, self.photoCalibCol] 

1512 

1513 @property 

1514 def name(self): 

1515 return f'flux_{self.instFluxCol}' 

1516 

1517 def _func(self, df): 

1518 return self.instFluxToNanojansky(df[self.instFluxCol], df[self.photoCalibCol]) 

1519 

1520 

1521class LocalNanojanskyErr(LocalPhotometry): 

1522 """Compute calibrated flux errors using the local calibration value. 

1523 

1524 See also 

1525 -------- 

1526 LocalNanojansky 

1527 LocalNanojanskyErr 

1528 LocalMagnitude 

1529 LocalMagnitudeErr 

1530 """ 

1531 

1532 @property 

1533 def columns(self): 

1534 return [self.instFluxCol, self.instFluxErrCol, 

1535 self.photoCalibCol, self.photoCalibErrCol] 

1536 

1537 @property 

1538 def name(self): 

1539 return f'fluxErr_{self.instFluxCol}' 

1540 

1541 def _func(self, df): 

1542 return self.instFluxErrToNanojanskyErr(df[self.instFluxCol], df[self.instFluxErrCol], 

1543 df[self.photoCalibCol], df[self.photoCalibErrCol]) 

1544 

1545 

1546class LocalMagnitude(LocalPhotometry): 

1547 """Compute calibrated AB magnitudes using the local calibration value. 

1548 

1549 See also 

1550 -------- 

1551 LocalNanojansky 

1552 LocalNanojanskyErr 

1553 LocalMagnitude 

1554 LocalMagnitudeErr 

1555 """ 

1556 

1557 @property 

1558 def columns(self): 

1559 return [self.instFluxCol, self.photoCalibCol] 

1560 

1561 @property 

1562 def name(self): 

1563 return f'mag_{self.instFluxCol}' 

1564 

1565 def _func(self, df): 

1566 return self.instFluxToMagnitude(df[self.instFluxCol], 

1567 df[self.photoCalibCol]) 

1568 

1569 

1570class LocalMagnitudeErr(LocalPhotometry): 

1571 """Compute calibrated AB magnitude errors using the local calibration value. 

1572 

1573 See also 

1574 -------- 

1575 LocalNanojansky 

1576 LocalNanojanskyErr 

1577 LocalMagnitude 

1578 LocalMagnitudeErr 

1579 """ 

1580 

1581 @property 

1582 def columns(self): 

1583 return [self.instFluxCol, self.instFluxErrCol, 

1584 self.photoCalibCol, self.photoCalibErrCol] 

1585 

1586 @property 

1587 def name(self): 

1588 return f'magErr_{self.instFluxCol}' 

1589 

1590 def _func(self, df): 

1591 return self.instFluxErrToMagnitudeErr(df[self.instFluxCol], 

1592 df[self.instFluxErrCol], 

1593 df[self.photoCalibCol], 

1594 df[self.photoCalibErrCol])