lsst.pipe.tasks  21.0.0-41-g0a6ae792+3de7980432
functors.py
Go to the documentation of this file.
1 import yaml
2 import re
3 from itertools import product
4 
5 import pandas as pd
6 import numpy as np
7 import astropy.units as u
8 
9 from lsst.daf.persistence import doImport
10 from lsst.daf.butler import DeferredDatasetHandle
11 from .parquetTable import ParquetTable, MultilevelParquetTable
12 
13 
14 def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors',
15  typeKey='functor', name=None):
16  """Initialize an object defined in a dictionary
17 
18  The object needs to be importable as
19  f'{basePath}.{initDict[typeKey]}'
20  The positional and keyword arguments (if any) are contained in
21  "args" and "kwargs" entries in the dictionary, respectively.
22  This is used in `functors.CompositeFunctor.from_yaml` to initialize
23  a composite functor from a specification in a YAML file.
24 
25  Parameters
26  ----------
27  initDict : dictionary
28  Dictionary describing object's initialization. Must contain
29  an entry keyed by ``typeKey`` that is the name of the object,
30  relative to ``basePath``.
31  basePath : str
32  Path relative to module in which ``initDict[typeKey]`` is defined.
33  typeKey : str
34  Key of ``initDict`` that is the name of the object
35  (relative to `basePath`).
36  """
37  initDict = initDict.copy()
38  # TO DO: DM-21956 We should be able to define functors outside this module
39  pythonType = doImport(f'{basePath}.{initDict.pop(typeKey)}')
40  args = []
41  if 'args' in initDict:
42  args = initDict.pop('args')
43  if isinstance(args, str):
44  args = [args]
45  try:
46  element = pythonType(*args, **initDict)
47  except Exception as e:
48  message = f'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
49  raise type(e)(message, e.args)
50  return element
51 
52 
53 class Functor(object):
54  """Define and execute a calculation on a ParquetTable
55 
56  The `__call__` method accepts either a `ParquetTable` object or a
57  `DeferredDatasetHandle`, and returns the
58  result of the calculation as a single column. Each functor defines what
59  columns are needed for the calculation, and only these columns are read
60  from the `ParquetTable`.
61 
62  The action of `__call__` consists of two steps: first, loading the
63  necessary columns from disk into memory as a `pandas.DataFrame` object;
64  and second, performing the computation on this dataframe and returning the
65  result.
66 
67 
68  To define a new `Functor`, a subclass must define a `_func` method,
69  that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
70  In addition, it must define the following attributes
71 
72  * `_columns`: The columns necessary to perform the calculation
73  * `name`: A name appropriate for a figure axis label
74  * `shortname`: A name appropriate for use as a dictionary key
75 
76  On initialization, a `Functor` should declare what band (`filt` kwarg)
77  and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
78  applied to. This enables the `_get_data` method to extract the proper
79  columns from the parquet file. If not specified, the dataset will fall back
80  on the `_defaultDataset`attribute. If band is not specified and `dataset`
81  is anything other than `'ref'`, then an error will be raised when trying to
82  perform the calculation.
83 
84  As currently implemented, `Functor` is only set up to expect a
85  dataset of the format of the `deepCoadd_obj` dataset; that is, a
86  dataframe with a multi-level column index,
87  with the levels of the column index being `band`,
88  `dataset`, and `column`. This is defined in the `_columnLevels` attribute,
89  as well as being implicit in the role of the `filt` and `dataset` attributes
90  defined at initialization. In addition, the `_get_data` method that reads
91  the dataframe from the `ParquetTable` will return a dataframe with column
92  index levels defined by the `_dfLevels` attribute; by default, this is
93  `column`.
94 
95  The `_columnLevels` and `_dfLevels` attributes should generally not need to
96  be changed, unless `_func` needs columns from multiple filters or datasets
97  to do the calculation.
98  An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
99  which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
100  it gets to have those levels in the column index.
101 
102  Parameters
103  ----------
104  filt : str
105  Filter upon which to do the calculation
106 
107  dataset : str
108  Dataset upon which to do the calculation
109  (e.g., 'ref', 'meas', 'forced_src').
110 
111  """
112 
113  _defaultDataset = 'ref'
114  _columnLevels = ('band', 'dataset', 'column')
115  _dfLevels = ('column',)
116  _defaultNoDup = False
117 
118  def __init__(self, filt=None, dataset=None, noDup=None):
119  self.filtfilt = filt
120  self.datasetdataset = dataset if dataset is not None else self._defaultDataset_defaultDataset
121  self._noDup_noDup = noDup
122 
123  @property
124  def noDup(self):
125  if self._noDup_noDup is not None:
126  return self._noDup_noDup
127  else:
128  return self._defaultNoDup_defaultNoDup
129 
130  @property
131  def columns(self):
132  """Columns required to perform calculation
133  """
134  if not hasattr(self, '_columns'):
135  raise NotImplementedError('Must define columns property or _columns attribute')
136  return self._columns
137 
138  def _get_data_columnLevels(self, data, columnIndex=None):
139  """Gets the names of the column index levels
140 
141  This should only be called in the context of a multilevel table.
142  The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
143  and with the gen3 `DeferredDatasetHandle`.
144 
145  Parameters
146  ----------
147  data : `MultilevelParquetTable` or `DeferredDatasetHandle`
148 
149  columnnIndex (optional): pandas `Index` object
150  if not passed, then it is read from the `DeferredDatasetHandle`
151  """
152  if isinstance(data, DeferredDatasetHandle):
153  if columnIndex is None:
154  columnIndex = data.get(component="columns")
155  if columnIndex is not None:
156  return columnIndex.names
157  if isinstance(data, MultilevelParquetTable):
158  return data.columnLevels
159  else:
160  raise TypeError(f"Unknown type for data: {type(data)}!")
161 
162  def _get_data_columnLevelNames(self, data, columnIndex=None):
163  """Gets the content of each of the column levels for a multilevel table
164 
165  Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
166 
167  Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
168  """
169  if isinstance(data, DeferredDatasetHandle):
170  if columnIndex is None:
171  columnIndex = data.get(component="columns")
172  if columnIndex is not None:
173  columnLevels = columnIndex.names
174  columnLevelNames = {
175  level: list(np.unique(np.array([c for c in columnIndex])[:, i]))
176  for i, level in enumerate(columnLevels)
177  }
178  return columnLevelNames
179  if isinstance(data, MultilevelParquetTable):
180  return data.columnLevelNames
181  else:
182  raise TypeError(f"Unknown type for data: {type(data)}!")
183 
184  def _colsFromDict(self, colDict, columnIndex=None):
185  """Converts dictionary column specficiation to a list of columns
186 
187  This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
188  """
189  new_colDict = {}
190  columnLevels = self._get_data_columnLevels_get_data_columnLevels(None, columnIndex=columnIndex)
191 
192  for i, lev in enumerate(columnLevels):
193  if lev in colDict:
194  if isinstance(colDict[lev], str):
195  new_colDict[lev] = [colDict[lev]]
196  else:
197  new_colDict[lev] = colDict[lev]
198  else:
199  new_colDict[lev] = columnIndex.levels[i]
200 
201  levelCols = [new_colDict[lev] for lev in columnLevels]
202  cols = product(*levelCols)
203  return list(cols)
204 
205  def multilevelColumns(self, data, columnIndex=None, returnTuple=False):
206  """Returns columns needed by functor from multilevel dataset
207 
208  To access tables with multilevel column structure, the `MultilevelParquetTable`
209  or `DeferredDatasetHandle` need to be passed either a list of tuples or a
210  dictionary.
211 
212  Parameters
213  ----------
214  data : `MultilevelParquetTable` or `DeferredDatasetHandle`
215 
216  columnIndex (optional): pandas `Index` object
217  either passed or read in from `DeferredDatasetHandle`.
218 
219  `returnTuple` : bool
220  If true, then return a list of tuples rather than the column dictionary
221  specification. This is set to `True` by `CompositeFunctor` in order to be able to
222  combine columns from the various component functors.
223 
224  """
225  if isinstance(data, DeferredDatasetHandle) and columnIndex is None:
226  columnIndex = data.get(component="columns")
227 
228  # Confirm that the dataset has the column levels the functor is expecting it to have.
229  columnLevels = self._get_data_columnLevels_get_data_columnLevels(data, columnIndex)
230 
231  if not set(columnLevels) == set(self._columnLevels_columnLevels):
232  raise ValueError(
233  "ParquetTable does not have the expected column levels. "
234  f"Got {columnLevels}; expected {self._columnLevels}."
235  )
236 
237  columnDict = {'column': self.columnscolumns,
238  'dataset': self.datasetdataset}
239  if self.filtfilt is None:
240  columnLevelNames = self._get_data_columnLevelNames_get_data_columnLevelNames(data, columnIndex)
241  if "band" in columnLevels:
242  if self.datasetdataset == "ref":
243  columnDict["band"] = columnLevelNames["band"][0]
244  else:
245  raise ValueError(f"'filt' not set for functor {self.name}"
246  f"(dataset {self.dataset}) "
247  "and ParquetTable "
248  "contains multiple filters in column index. "
249  "Set 'filt' or set 'dataset' to 'ref'.")
250  else:
251  columnDict['band'] = self.filtfilt
252 
253  if isinstance(data, MultilevelParquetTable):
254  return data._colsFromDict(columnDict)
255  elif isinstance(data, DeferredDatasetHandle):
256  if returnTuple:
257  return self._colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
258  else:
259  return columnDict
260 
261  def _func(self, df, dropna=True):
262  raise NotImplementedError('Must define calculation on dataframe')
263 
264  def _get_columnIndex(self, data):
265  """Return columnIndex
266  """
267 
268  if isinstance(data, DeferredDatasetHandle):
269  return data.get(component="columns")
270  else:
271  return None
272 
273  def _get_data(self, data):
274  """Retrieve dataframe necessary for calculation.
275 
276  The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
277 
278  Returns dataframe upon which `self._func` can act.
279 
280  N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
281  """
282  if isinstance(data, pd.DataFrame):
283  return data
284 
285  # First thing to do: check to see if the data source has a multilevel column index or not.
286  columnIndex = self._get_columnIndex_get_columnIndex(data)
287  is_multiLevel = isinstance(data, MultilevelParquetTable) or isinstance(columnIndex, pd.MultiIndex)
288 
289  # Simple single-level parquet table, gen2
290  if isinstance(data, ParquetTable) and not is_multiLevel:
291  columns = self.columnscolumns
292  df = data.toDataFrame(columns=columns)
293  return df
294 
295  # Get proper multi-level columns specification for this functor
296  if is_multiLevel:
297  columns = self.multilevelColumnsmultilevelColumns(data, columnIndex=columnIndex)
298 
299  if isinstance(data, MultilevelParquetTable):
300  # Load in-memory dataframe with appropriate columns the gen2 way
301  df = data.toDataFrame(columns=columns, droplevels=False)
302  elif isinstance(data, DeferredDatasetHandle):
303  # Load in-memory dataframe with appropriate columns the gen3 way
304  df = data.get(parameters={"columns": columns})
305 
306  # Drop unnecessary column levels
307  df = self._setLevels_setLevels(df)
308  return df
309 
310  def _setLevels(self, df):
311  levelsToDrop = [n for n in df.columns.names if n not in self._dfLevels_dfLevels]
312  df.columns = df.columns.droplevel(levelsToDrop)
313  return df
314 
315  def _dropna(self, vals):
316  return vals.dropna()
317 
318  def __call__(self, data, dropna=False):
319  try:
320  df = self._get_data_get_data(data)
321  vals = self._func_func(df)
322  except Exception:
323  vals = self.failfail(df)
324  if dropna:
325  vals = self._dropna_dropna(vals)
326 
327  return vals
328 
329  def difference(self, data1, data2, **kwargs):
330  """Computes difference between functor called on two different ParquetTable objects
331  """
332  return self(data1, **kwargs) - self(data2, **kwargs)
333 
334  def fail(self, df):
335  return pd.Series(np.full(len(df), np.nan), index=df.index)
336 
337  @property
338  def name(self):
339  """Full name of functor (suitable for figure labels)
340  """
341  return NotImplementedError
342 
343  @property
344  def shortname(self):
345  """Short name of functor (suitable for column name/dict key)
346  """
347  return self.namename
348 
349 
351  """Perform multiple calculations at once on a catalog
352 
353  The role of a `CompositeFunctor` is to group together computations from
354  multiple functors. Instead of returning `pandas.Series` a
355  `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
356  being the keys of `funcDict`.
357 
358  The `columns` attribute of a `CompositeFunctor` is the union of all columns
359  in all the component functors.
360 
361  A `CompositeFunctor` does not use a `_func` method itself; rather,
362  when a `CompositeFunctor` is called, all its columns are loaded
363  at once, and the resulting dataframe is passed to the `_func` method of each component
364  functor. This has the advantage of only doing I/O (reading from parquet file) once,
365  and works because each individual `_func` method of each component functor does not
366  care if there are *extra* columns in the dataframe being passed; only that it must contain
367  *at least* the `columns` it expects.
368 
369  An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
370  file specifying a collection of functors.
371 
372  Parameters
373  ----------
374  funcs : `dict` or `list`
375  Dictionary or list of functors. If a list, then it will be converted
376  into a dictonary according to the `.shortname` attribute of each functor.
377 
378  """
379  dataset = None
380 
381  def __init__(self, funcs, **kwargs):
382 
383  if type(funcs) == dict:
384  self.funcDictfuncDict = funcs
385  else:
386  self.funcDictfuncDict = {f.shortname: f for f in funcs}
387 
388  self._filt_filt = None
389 
390  super().__init__(**kwargs)
391 
392  @property
393  def filt(self):
394  return self._filt_filt
395 
396  @filt.setter
397  def filt(self, filt):
398  if filt is not None:
399  for _, f in self.funcDictfuncDict.items():
400  f.filt = filt
401  self._filt_filt = filt
402 
403  def update(self, new):
404  if isinstance(new, dict):
405  self.funcDictfuncDict.update(new)
406  elif isinstance(new, CompositeFunctor):
407  self.funcDictfuncDict.update(new.funcDict)
408  else:
409  raise TypeError('Can only update with dictionary or CompositeFunctor.')
410 
411  # Make sure new functors have the same 'filt' set
412  if self.filtfiltfiltfiltfilt is not None:
413  self.filtfiltfiltfiltfilt = self.filtfiltfiltfiltfilt
414 
415  @property
416  def columns(self):
417  return list(set([x for y in [f.columns for f in self.funcDictfuncDict.values()] for x in y]))
418 
419  def multilevelColumns(self, data, **kwargs):
420  # Get the union of columns for all component functors. Note the need to have `returnTuple=True` here.
421  return list(
422  set(
423  [
424  x
425  for y in [
426  f.multilevelColumns(data, returnTuple=True, **kwargs) for f in self.funcDictfuncDict.values()
427  ]
428  for x in y
429  ]
430  )
431  )
432 
433  def __call__(self, data, **kwargs):
434  """Apply the functor to the data table
435 
436  Parameters
437  ----------
438  data : `lsst.daf.butler.DeferredDatasetHandle`,
439  `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
440  `lsst.pipe.tasks.parquetTable.ParquetTable`,
441  or `pandas.DataFrame`.
442  The table or a pointer to a table on disk from which columns can
443  be accessed
444  """
445  columnIndex = self._get_columnIndex_get_columnIndex(data)
446 
447  # First, determine whether data has a multilevel index (either gen2 or gen3)
448  is_multiLevel = isinstance(data, MultilevelParquetTable) or isinstance(columnIndex, pd.MultiIndex)
449 
450  # Multilevel index, gen2 or gen3
451  if is_multiLevel:
452  columns = self.multilevelColumnsmultilevelColumnsmultilevelColumns(data, columnIndex=columnIndex)
453 
454  if isinstance(data, MultilevelParquetTable):
455  # Read data into memory the gen2 way
456  df = data.toDataFrame(columns=columns, droplevels=False)
457  elif isinstance(data, DeferredDatasetHandle):
458  # Read data into memory the gen3 way
459  df = data.get(parameters={"columns": columns})
460 
461  valDict = {}
462  for k, f in self.funcDictfuncDict.items():
463  try:
464  subdf = f._setLevels(
465  df[f.multilevelColumns(data, returnTuple=True, columnIndex=columnIndex)]
466  )
467  valDict[k] = f._func(subdf)
468  except Exception:
469  valDict[k] = f.fail(subdf)
470 
471  else:
472  if isinstance(data, DeferredDatasetHandle):
473  # input if Gen3 deferLoad=True
474  df = data.get(columns=self.columnscolumnscolumns)
475  elif isinstance(data, pd.DataFrame):
476  # input if Gen3 deferLoad=False
477  df = data
478  else:
479  # Original Gen2 input is type ParquetTable and the fallback
480  df = data.toDataFrame(columns=self.columnscolumnscolumns)
481 
482  valDict = {k: f._func(df) for k, f in self.funcDictfuncDict.items()}
483 
484  try:
485  valDf = pd.concat(valDict, axis=1)
486  except TypeError:
487  print([(k, type(v)) for k, v in valDict.items()])
488  raise
489 
490  if kwargs.get('dropna', False):
491  valDf = valDf.dropna(how='any')
492 
493  return valDf
494 
495  @classmethod
496  def renameCol(cls, col, renameRules):
497  if renameRules is None:
498  return col
499  for old, new in renameRules:
500  if col.startswith(old):
501  col = col.replace(old, new)
502  return col
503 
504  @classmethod
505  def from_file(cls, filename, **kwargs):
506  with open(filename) as f:
507  translationDefinition = yaml.safe_load(f)
508 
509  return cls.from_yamlfrom_yaml(translationDefinition, **kwargs)
510 
511  @classmethod
512  def from_yaml(cls, translationDefinition, **kwargs):
513  funcs = {}
514  for func, val in translationDefinition['funcs'].items():
515  funcs[func] = init_fromDict(val, name=func)
516 
517  if 'flag_rename_rules' in translationDefinition:
518  renameRules = translationDefinition['flag_rename_rules']
519  else:
520  renameRules = None
521 
522  if 'refFlags' in translationDefinition:
523  for flag in translationDefinition['refFlags']:
524  funcs[cls.renameColrenameCol(flag, renameRules)] = Column(flag, dataset='ref')
525 
526  if 'flags' in translationDefinition:
527  for flag in translationDefinition['flags']:
528  funcs[cls.renameColrenameCol(flag, renameRules)] = Column(flag, dataset='meas')
529 
530  return cls(funcs, **kwargs)
531 
532 
533 def mag_aware_eval(df, expr):
534  """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
535 
536  Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
537 
538  Parameters
539  ----------
540  df : pandas.DataFrame
541  Dataframe on which to evaluate expression.
542 
543  expr : str
544  Expression.
545  """
546  try:
547  expr_new = re.sub(r'mag\‍((\w+)\‍)', r'-2.5*log(\g<1>)/log(10)', expr)
548  val = df.eval(expr_new, truediv=True)
549  except Exception: # Should check what actually gets raised
550  expr_new = re.sub(r'mag\‍((\w+)\‍)', r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
551  val = df.eval(expr_new, truediv=True)
552  return val
553 
554 
556  """Arbitrary computation on a catalog
557 
558  Column names (and thus the columns to be loaded from catalog) are found
559  by finding all words and trying to ignore all "math-y" words.
560 
561  Parameters
562  ----------
563  expr : str
564  Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
565  """
566  _ignore_words = ('mag', 'sin', 'cos', 'exp', 'log', 'sqrt')
567 
568  def __init__(self, expr, **kwargs):
569  self.exprexpr = expr
570  super().__init__(**kwargs)
571 
572  @property
573  def name(self):
574  return self.exprexpr
575 
576  @property
577  def columns(self):
578  flux_cols = re.findall(r'mag\‍(\s*(\w+)\s*\‍)', self.exprexpr)
579 
580  cols = [c for c in re.findall(r'[a-zA-Z_]+', self.exprexpr) if c not in self._ignore_words_ignore_words]
581  not_a_col = []
582  for c in flux_cols:
583  if not re.search('_instFlux$', c):
584  cols.append(f'{c}_instFlux')
585  not_a_col.append(c)
586  else:
587  cols.append(c)
588 
589  return list(set([c for c in cols if c not in not_a_col]))
590 
591  def _func(self, df):
592  return mag_aware_eval(df, self.exprexpr)
593 
594 
596  """Get column with specified name
597  """
598 
599  def __init__(self, col, **kwargs):
600  self.colcol = col
601  super().__init__(**kwargs)
602 
603  @property
604  def name(self):
605  return self.colcol
606 
607  @property
608  def columns(self):
609  return [self.colcol]
610 
611  def _func(self, df):
612  return df[self.colcol]
613 
614 
615 class Index(Functor):
616  """Return the value of the index for each object
617  """
618 
619  columns = ['coord_ra'] # just a dummy; something has to be here
620  _defaultDataset = 'ref'
621  _defaultNoDup = True
622 
623  def _func(self, df):
624  return pd.Series(df.index, index=df.index)
625 
626 
628  col = 'id'
629  _allow_difference = False
630  _defaultNoDup = True
631 
632  def _func(self, df):
633  return pd.Series(df.index, index=df.index)
634 
635 
637  col = 'base_Footprint_nPix'
638 
639 
641  """Base class for coordinate column, in degrees
642  """
643  _radians = True
644 
645  def __init__(self, col, **kwargs):
646  super().__init__(col, **kwargs)
647 
648  def _func(self, df):
649  # Must not modify original column in case that column is used by another functor
650  output = df[self.colcol] * 180 / np.pi if self._radians_radians else df[self.colcol]
651  return output
652 
653 
655  """Right Ascension, in degrees
656  """
657  name = 'RA'
658  _defaultNoDup = True
659 
660  def __init__(self, **kwargs):
661  super().__init__('coord_ra', **kwargs)
662 
663  def __call__(self, catalog, **kwargs):
664  return super().__call__(catalog, **kwargs)
665 
666 
668  """Declination, in degrees
669  """
670  name = 'Dec'
671  _defaultNoDup = True
672 
673  def __init__(self, **kwargs):
674  super().__init__('coord_dec', **kwargs)
675 
676  def __call__(self, catalog, **kwargs):
677  return super().__call__(catalog, **kwargs)
678 
679 
680 def fluxName(col):
681  if not col.endswith('_instFlux'):
682  col += '_instFlux'
683  return col
684 
685 
686 def fluxErrName(col):
687  if not col.endswith('_instFluxErr'):
688  col += '_instFluxErr'
689  return col
690 
691 
692 class Mag(Functor):
693  """Compute calibrated magnitude
694 
695  Takes a `calib` argument, which returns the flux at mag=0
696  as `calib.getFluxMag0()`. If not provided, then the default
697  `fluxMag0` is 63095734448.0194, which is default for HSC.
698  This default should be removed in DM-21955
699 
700  This calculation hides warnings about invalid values and dividing by zero.
701 
702  As for all functors, a `dataset` and `filt` kwarg should be provided upon
703  initialization. Unlike the default `Functor`, however, the default dataset
704  for a `Mag` is `'meas'`, rather than `'ref'`.
705 
706  Parameters
707  ----------
708  col : `str`
709  Name of flux column from which to compute magnitude. Can be parseable
710  by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
711  `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
712  understand.
713  calib : `lsst.afw.image.calib.Calib` (optional)
714  Object that knows zero point.
715  """
716  _defaultDataset = 'meas'
717 
718  def __init__(self, col, calib=None, **kwargs):
719  self.colcol = fluxName(col)
720  self.calibcalib = calib
721  if calib is not None:
722  self.fluxMag0fluxMag0 = calib.getFluxMag0()[0]
723  else:
724  # TO DO: DM-21955 Replace hard coded photometic calibration values
725  self.fluxMag0fluxMag0 = 63095734448.0194
726 
727  super().__init__(**kwargs)
728 
729  @property
730  def columns(self):
731  return [self.colcol]
732 
733  def _func(self, df):
734  with np.warnings.catch_warnings():
735  np.warnings.filterwarnings('ignore', r'invalid value encountered')
736  np.warnings.filterwarnings('ignore', r'divide by zero')
737  return -2.5*np.log10(df[self.colcol] / self.fluxMag0fluxMag0)
738 
739  @property
740  def name(self):
741  return f'mag_{self.col}'
742 
743 
744 class MagErr(Mag):
745  """Compute calibrated magnitude uncertainty
746 
747  Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
748 
749  Parameters
750  col : `str`
751  Name of flux column
752  calib : `lsst.afw.image.calib.Calib` (optional)
753  Object that knows zero point.
754  """
755 
756  def __init__(self, *args, **kwargs):
757  super().__init__(*args, **kwargs)
758  if self.calibcalib is not None:
759  self.fluxMag0ErrfluxMag0Err = self.calibcalib.getFluxMag0()[1]
760  else:
761  self.fluxMag0ErrfluxMag0Err = 0.
762 
763  @property
764  def columns(self):
765  return [self.colcol, self.colcol + 'Err']
766 
767  def _func(self, df):
768  with np.warnings.catch_warnings():
769  np.warnings.filterwarnings('ignore', r'invalid value encountered')
770  np.warnings.filterwarnings('ignore', r'divide by zero')
771  fluxCol, fluxErrCol = self.columnscolumnscolumnscolumns
772  x = df[fluxErrCol] / df[fluxCol]
773  y = self.fluxMag0ErrfluxMag0Err / self.fluxMag0fluxMag0
774  magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
775  return magErr
776 
777  @property
778  def name(self):
779  return super().name + '_err'
780 
781 
783  """
784  """
785 
786  def _func(self, df):
787  return (df[self.colcol] / self.fluxMag0fluxMag0) * 1e9
788 
789 
791  _defaultDataset = 'meas'
792 
793  """Functor to calculate magnitude difference"""
794 
795  def __init__(self, col1, col2, **kwargs):
796  self.col1col1 = fluxName(col1)
797  self.col2col2 = fluxName(col2)
798  super().__init__(**kwargs)
799 
800  @property
801  def columns(self):
802  return [self.col1col1, self.col2col2]
803 
804  def _func(self, df):
805  with np.warnings.catch_warnings():
806  np.warnings.filterwarnings('ignore', r'invalid value encountered')
807  np.warnings.filterwarnings('ignore', r'divide by zero')
808  return -2.5*np.log10(df[self.col1col1]/df[self.col2col2])
809 
810  @property
811  def name(self):
812  return f'(mag_{self.col1} - mag_{self.col2})'
813 
814  @property
815  def shortname(self):
816  return f'magDiff_{self.col1}_{self.col2}'
817 
818 
819 class Color(Functor):
820  """Compute the color between two filters
821 
822  Computes color by initializing two different `Mag`
823  functors based on the `col` and filters provided, and
824  then returning the difference.
825 
826  This is enabled by the `_func` expecting a dataframe with a
827  multilevel column index, with both `'band'` and `'column'`,
828  instead of just `'column'`, which is the `Functor` default.
829  This is controlled by the `_dfLevels` attribute.
830 
831  Also of note, the default dataset for `Color` is `forced_src'`,
832  whereas for `Mag` it is `'meas'`.
833 
834  Parameters
835  ----------
836  col : str
837  Name of flux column from which to compute; same as would be passed to
838  `lsst.pipe.tasks.functors.Mag`.
839 
840  filt2, filt1 : str
841  Filters from which to compute magnitude difference.
842  Color computed is `Mag(filt2) - Mag(filt1)`.
843  """
844  _defaultDataset = 'forced_src'
845  _dfLevels = ('band', 'column')
846  _defaultNoDup = True
847 
848  def __init__(self, col, filt2, filt1, **kwargs):
849  self.colcol = fluxName(col)
850  if filt2 == filt1:
851  raise RuntimeError("Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
852  self.filt2filt2 = filt2
853  self.filt1filt1 = filt1
854 
855  self.mag2mag2 = Mag(col, filt=filt2, **kwargs)
856  self.mag1mag1 = Mag(col, filt=filt1, **kwargs)
857 
858  super().__init__(**kwargs)
859 
860  @property
861  def filt(self):
862  return None
863 
864  @filt.setter
865  def filt(self, filt):
866  pass
867 
868  def _func(self, df):
869  mag2 = self.mag2._func(df[self.filt2])
870  mag1 = self.mag1._func(df[self.filt1])
871  return mag2 - mag1
872 
873  @property
874  def columns(self):
875  return [self.mag1mag1.col, self.mag2mag2.col]
876 
877  def multilevelColumns(self, parq, **kwargs):
878  return [(self.datasetdataset, self.filt1filt1, self.colcol), (self.datasetdataset, self.filt2filt2, self.colcol)]
879 
880  @property
881  def name(self):
882  return f'{self.filt2} - {self.filt1} ({self.col})'
883 
884  @property
885  def shortname(self):
886  return f"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
887 
888 
890  """Main function of this subclass is to override the dropna=True
891  """
892  _null_label = 'null'
893  _allow_difference = False
894  name = 'label'
895  _force_str = False
896 
897  def __call__(self, parq, dropna=False, **kwargs):
898  return super().__call__(parq, dropna=False, **kwargs)
899 
900 
902  _columns = ["base_ClassificationExtendedness_value"]
903  _column = "base_ClassificationExtendedness_value"
904 
905  def _func(self, df):
906  x = df[self._columns_columns][self._column_column]
907  mask = x.isnull()
908  test = (x < 0.5).astype(int)
909  test = test.mask(mask, 2)
910 
911  # TODO: DM-21954 Look into veracity of inline comment below
912  # are these backwards?
913  categories = ['galaxy', 'star', self._null_label_null_label]
914  label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
915  index=x.index, name='label')
916  if self._force_str_force_str:
917  label = label.astype(str)
918  return label
919 
920 
922  _columns = ['numStarFlags']
923  labels = {"star": 0, "maybe": 1, "notStar": 2}
924 
925  def _func(self, df):
926  x = df[self._columns_columns][self._columns_columns[0]]
927 
928  # Number of filters
929  n = len(x.unique()) - 1
930 
931  labels = ['noStar', 'maybe', 'star']
932  label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
933  index=x.index, name='label')
934 
935  if self._force_str_force_str:
936  label = label.astype(str)
937 
938  return label
939 
940 
942  name = 'Deconvolved Moments'
943  shortname = 'deconvolvedMoments'
944  _columns = ("ext_shapeHSM_HsmSourceMoments_xx",
945  "ext_shapeHSM_HsmSourceMoments_yy",
946  "base_SdssShape_xx", "base_SdssShape_yy",
947  "ext_shapeHSM_HsmPsfMoments_xx",
948  "ext_shapeHSM_HsmPsfMoments_yy")
949 
950  def _func(self, df):
951  """Calculate deconvolved moments"""
952  if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns: # _xx added by tdm
953  hsm = df["ext_shapeHSM_HsmSourceMoments_xx"] + df["ext_shapeHSM_HsmSourceMoments_yy"]
954  else:
955  hsm = np.ones(len(df))*np.nan
956  sdss = df["base_SdssShape_xx"] + df["base_SdssShape_yy"]
957  if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
958  psf = df["ext_shapeHSM_HsmPsfMoments_xx"] + df["ext_shapeHSM_HsmPsfMoments_yy"]
959  else:
960  # LSST does not have shape.sdss.psf. Could instead add base_PsfShape to catalog using
961  # exposure.getPsf().computeShape(s.getCentroid()).getIxx()
962  # raise TaskError("No psf shape parameter found in catalog")
963  raise RuntimeError('No psf shape parameter found in catalog')
964 
965  return hsm.where(np.isfinite(hsm), sdss) - psf
966 
967 
969  """Functor to calculate SDSS trace radius size for sources"""
970  name = "SDSS Trace Size"
971  shortname = 'sdssTrace'
972  _columns = ("base_SdssShape_xx", "base_SdssShape_yy")
973 
974  def _func(self, df):
975  srcSize = np.sqrt(0.5*(df["base_SdssShape_xx"] + df["base_SdssShape_yy"]))
976  return srcSize
977 
978 
980  """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
981  name = "PSF - SDSS Trace Size"
982  shortname = 'psf_sdssTrace'
983  _columns = ("base_SdssShape_xx", "base_SdssShape_yy",
984  "base_SdssShape_psf_xx", "base_SdssShape_psf_yy")
985 
986  def _func(self, df):
987  srcSize = np.sqrt(0.5*(df["base_SdssShape_xx"] + df["base_SdssShape_yy"]))
988  psfSize = np.sqrt(0.5*(df["base_SdssShape_psf_xx"] + df["base_SdssShape_psf_yy"]))
989  sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
990  return sizeDiff
991 
992 
994  """Functor to calculate HSM trace radius size for sources"""
995  name = 'HSM Trace Size'
996  shortname = 'hsmTrace'
997  _columns = ("ext_shapeHSM_HsmSourceMoments_xx",
998  "ext_shapeHSM_HsmSourceMoments_yy")
999 
1000  def _func(self, df):
1001  srcSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmSourceMoments_xx"]
1002  + df["ext_shapeHSM_HsmSourceMoments_yy"]))
1003  return srcSize
1004 
1005 
1007  """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1008  name = 'PSF - HSM Trace Size'
1009  shortname = 'psf_HsmTrace'
1010  _columns = ("ext_shapeHSM_HsmSourceMoments_xx",
1011  "ext_shapeHSM_HsmSourceMoments_yy",
1012  "ext_shapeHSM_HsmPsfMoments_xx",
1013  "ext_shapeHSM_HsmPsfMoments_yy")
1014 
1015  def _func(self, df):
1016  srcSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmSourceMoments_xx"]
1017  + df["ext_shapeHSM_HsmSourceMoments_yy"]))
1018  psfSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmPsfMoments_xx"]
1019  + df["ext_shapeHSM_HsmPsfMoments_yy"]))
1020  sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1021  return sizeDiff
1022 
1023 
1025  name = 'HSM Psf FWHM'
1026  _columns = ('ext_shapeHSM_HsmPsfMoments_xx', 'ext_shapeHSM_HsmPsfMoments_yy')
1027  # TODO: DM-21403 pixel scale should be computed from the CD matrix or transform matrix
1028  pixelScale = 0.168
1029  SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1030 
1031  def _func(self, df):
1032  return self.pixelScalepixelScale*self.SIGMA2FWHMSIGMA2FWHM*np.sqrt(
1033  0.5*(df['ext_shapeHSM_HsmPsfMoments_xx'] + df['ext_shapeHSM_HsmPsfMoments_yy']))
1034 
1035 
1036 class E1(Functor):
1037  name = "Distortion Ellipticity (e1)"
1038  shortname = "Distortion"
1039 
1040  def __init__(self, colXX, colXY, colYY, **kwargs):
1041  self.colXXcolXX = colXX
1042  self.colXYcolXY = colXY
1043  self.colYYcolYY = colYY
1044  self._columns_columns = [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1045  super().__init__(**kwargs)
1046 
1047  @property
1048  def columns(self):
1049  return [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1050 
1051  def _func(self, df):
1052  return df[self.colXXcolXX] - df[self.colYYcolYY] / (df[self.colXXcolXX] + df[self.colYYcolYY])
1053 
1054 
1055 class E2(Functor):
1056  name = "Ellipticity e2"
1057 
1058  def __init__(self, colXX, colXY, colYY, **kwargs):
1059  self.colXXcolXX = colXX
1060  self.colXYcolXY = colXY
1061  self.colYYcolYY = colYY
1062  super().__init__(**kwargs)
1063 
1064  @property
1065  def columns(self):
1066  return [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1067 
1068  def _func(self, df):
1069  return 2*df[self.colXYcolXY] / (df[self.colXXcolXX] + df[self.colYYcolYY])
1070 
1071 
1073 
1074  def __init__(self, colXX, colXY, colYY, **kwargs):
1075  self.colXXcolXX = colXX
1076  self.colXYcolXY = colXY
1077  self.colYYcolYY = colYY
1078  super().__init__(**kwargs)
1079 
1080  @property
1081  def columns(self):
1082  return [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1083 
1084  def _func(self, df):
1085  return (df[self.colXXcolXX]*df[self.colYYcolYY] - df[self.colXYcolXY]**2)**0.25
1086 
1087 
1089  """Computations using the stored localWcs.
1090  """
1091  name = "LocalWcsOperations"
1092 
1093  def __init__(self,
1094  colCD_1_1,
1095  colCD_1_2,
1096  colCD_2_1,
1097  colCD_2_2,
1098  **kwargs):
1099  self.colCD_1_1colCD_1_1 = colCD_1_1
1100  self.colCD_1_2colCD_1_2 = colCD_1_2
1101  self.colCD_2_1colCD_2_1 = colCD_2_1
1102  self.colCD_2_2colCD_2_2 = colCD_2_2
1103  super().__init__(**kwargs)
1104 
1105  def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22):
1106  """Compute the distance on the sphere from x2, y1 to x1, y1.
1107 
1108  Parameters
1109  ----------
1110  x : `pandas.Series`
1111  X pixel coordinate.
1112  y : `pandas.Series`
1113  Y pixel coordinate.
1114  cd11 : `pandas.Series`
1115  [1, 1] element of the local Wcs affine transform.
1116  cd11 : `pandas.Series`
1117  [1, 1] element of the local Wcs affine transform.
1118  cd12 : `pandas.Series`
1119  [1, 2] element of the local Wcs affine transform.
1120  cd21 : `pandas.Series`
1121  [2, 1] element of the local Wcs affine transform.
1122  cd22 : `pandas.Series`
1123  [2, 2] element of the local Wcs affine transform.
1124 
1125  Returns
1126  -------
1127  raDecTuple : tuple
1128  RA and dec conversion of x and y given the local Wcs. Returned
1129  units are in radians.
1130 
1131  """
1132  return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1133 
1134  def computeSkySeperation(self, ra1, dec1, ra2, dec2):
1135  """Compute the local pixel scale conversion.
1136 
1137  Parameters
1138  ----------
1139  ra1 : `pandas.Series`
1140  Ra of the first coordinate in radians.
1141  dec1 : `pandas.Series`
1142  Dec of the first coordinate in radians.
1143  ra2 : `pandas.Series`
1144  Ra of the second coordinate in radians.
1145  dec2 : `pandas.Series`
1146  Dec of the second coordinate in radians.
1147 
1148  Returns
1149  -------
1150  dist : `pandas.Series`
1151  Distance on the sphere in radians.
1152  """
1153  deltaDec = dec2 - dec1
1154  deltaRa = ra2 - ra1
1155  return 2 * np.arcsin(
1156  np.sqrt(
1157  np.sin(deltaDec / 2) ** 2
1158  + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1159 
1160  def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22):
1161  """Compute the distance on the sphere from x2, y1 to x1, y1.
1162 
1163  Parameters
1164  ----------
1165  x1 : `pandas.Series`
1166  X pixel coordinate.
1167  y1 : `pandas.Series`
1168  Y pixel coordinate.
1169  x2 : `pandas.Series`
1170  X pixel coordinate.
1171  y2 : `pandas.Series`
1172  Y pixel coordinate.
1173  cd11 : `pandas.Series`
1174  [1, 1] element of the local Wcs affine transform.
1175  cd11 : `pandas.Series`
1176  [1, 1] element of the local Wcs affine transform.
1177  cd12 : `pandas.Series`
1178  [1, 2] element of the local Wcs affine transform.
1179  cd21 : `pandas.Series`
1180  [2, 1] element of the local Wcs affine transform.
1181  cd22 : `pandas.Series`
1182  [2, 2] element of the local Wcs affine transform.
1183 
1184  Returns
1185  -------
1186  Distance : `pandas.Series`
1187  Arcseconds per pixel at the location of the local WC
1188  """
1189  ra1, dec1 = self.computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1190  ra2, dec2 = self.computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1191  # Great circle distance for small separations.
1192  return self.computeSkySeperationcomputeSkySeperation(ra1, dec1, ra2, dec2)
1193 
1194 
1196  """Compute the local pixel scale from the stored CDMatrix.
1197  """
1198  name = "PixelScale"
1199 
1200  @property
1201  def columns(self):
1202  return [self.colCD_1_1colCD_1_1,
1203  self.colCD_1_2colCD_1_2,
1204  self.colCD_2_1colCD_2_1,
1205  self.colCD_2_2colCD_2_2]
1206 
1207  def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22):
1208  """Compute the local pixel to scale conversion in arcseconds.
1209 
1210  Parameters
1211  ----------
1212  cd11 : `pandas.Series`
1213  [1, 1] element of the local Wcs affine transform in radians.
1214  cd11 : `pandas.Series`
1215  [1, 1] element of the local Wcs affine transform in radians.
1216  cd12 : `pandas.Series`
1217  [1, 2] element of the local Wcs affine transform in radians.
1218  cd21 : `pandas.Series`
1219  [2, 1] element of the local Wcs affine transform in radians.
1220  cd22 : `pandas.Series`
1221  [2, 2] element of the local Wcs affine transform in radians.
1222 
1223  Returns
1224  -------
1225  pixScale : `pandas.Series`
1226  Arcseconds per pixel at the location of the local WC
1227  """
1228  return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1229 
1230  def _func(self, df):
1231  return self.pixelScaleArcsecondspixelScaleArcseconds(df[self.colCD_1_1colCD_1_1],
1232  df[self.colCD_1_2colCD_1_2],
1233  df[self.colCD_2_1colCD_2_1],
1234  df[self.colCD_2_2colCD_2_2])
1235 
1236 
1238  """Convert a value in units pixels to units arcseconds.
1239  """
1240 
1241  def __init__(self,
1242  col,
1243  colCD_1_1,
1244  colCD_1_2,
1245  colCD_2_1,
1246  colCD_2_2,
1247  **kwargs):
1248  self.colcol = col
1249  super().__init__(colCD_1_1,
1250  colCD_1_2,
1251  colCD_2_1,
1252  colCD_2_2,
1253  **kwargs)
1254 
1255  @property
1256  def name(self):
1257  return f"{self.col}_asArcseconds"
1258 
1259  @property
1260  def columns(self):
1261  return [self.colcol,
1262  self.colCD_1_1colCD_1_1,
1263  self.colCD_1_2colCD_1_2,
1264  self.colCD_2_1colCD_2_1,
1265  self.colCD_2_2colCD_2_2]
1266 
1267  def _func(self, df):
1268  return df[self.colcol] * self.pixelScaleArcsecondspixelScaleArcseconds(df[self.colCD_1_1colCD_1_1],
1269  df[self.colCD_1_2colCD_1_2],
1270  df[self.colCD_2_1colCD_2_1],
1271  df[self.colCD_2_2colCD_2_2])
1272 
1273 
1275  name = 'Reference Band'
1276  shortname = 'refBand'
1277 
1278  @property
1279  def columns(self):
1280  return ["merge_measurement_i",
1281  "merge_measurement_r",
1282  "merge_measurement_z",
1283  "merge_measurement_y",
1284  "merge_measurement_g"]
1285 
1286  def _func(self, df):
1287  def getFilterAliasName(row):
1288  # get column name with the max value (True > False)
1289  colName = row.idxmax()
1290  return colName.replace('merge_measurement_', '')
1291 
1292  return df[self.columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1293 
1294 
1296  # AB to NanoJansky (3631 Jansky)
1297  AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1298  LOG_AB_FLUX_SCALE = 12.56
1299  FIVE_OVER_2LOG10 = 1.085736204758129569
1300  # TO DO: DM-21955 Replace hard coded photometic calibration values
1301  COADD_ZP = 27
1302 
1303  def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1304  self.vhypotvhypot = np.vectorize(self.hypothypot)
1305  self.colcol = colFlux
1306  self.colFluxErrcolFluxErr = colFluxErr
1307 
1308  self.calibcalib = calib
1309  if calib is not None:
1310  self.fluxMag0fluxMag0, self.fluxMag0ErrfluxMag0Err = calib.getFluxMag0()
1311  else:
1312  self.fluxMag0fluxMag0 = 1./np.power(10, -0.4*self.COADD_ZPCOADD_ZP)
1313  self.fluxMag0ErrfluxMag0Err = 0.
1314 
1315  super().__init__(**kwargs)
1316 
1317  @property
1318  def columns(self):
1319  return [self.colcol]
1320 
1321  @property
1322  def name(self):
1323  return f'mag_{self.col}'
1324 
1325  @classmethod
1326  def hypot(cls, a, b):
1327  if np.abs(a) < np.abs(b):
1328  a, b = b, a
1329  if a == 0.:
1330  return 0.
1331  q = b/a
1332  return np.abs(a) * np.sqrt(1. + q*q)
1333 
1334  def dn2flux(self, dn, fluxMag0):
1335  return self.AB_FLUX_SCALEAB_FLUX_SCALE * dn / fluxMag0
1336 
1337  def dn2mag(self, dn, fluxMag0):
1338  with np.warnings.catch_warnings():
1339  np.warnings.filterwarnings('ignore', r'invalid value encountered')
1340  np.warnings.filterwarnings('ignore', r'divide by zero')
1341  return -2.5 * np.log10(dn/fluxMag0)
1342 
1343  def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err):
1344  retVal = self.vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1345  retVal *= self.AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1346  return retVal
1347 
1348  def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err):
1349  retVal = self.dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.dn2fluxdn2flux(dn, fluxMag0)
1350  return self.FIVE_OVER_2LOG10FIVE_OVER_2LOG10 * retVal
1351 
1352 
1354  def _func(self, df):
1355  return self.dn2fluxdn2flux(df[self.colcol], self.fluxMag0fluxMag0)
1356 
1357 
1359  @property
1360  def columns(self):
1361  return [self.colcol, self.colFluxErrcolFluxErr]
1362 
1363  def _func(self, df):
1364  retArr = self.dn2fluxErrdn2fluxErr(df[self.colcol], df[self.colFluxErrcolFluxErr], self.fluxMag0fluxMag0, self.fluxMag0ErrfluxMag0Err)
1365  return pd.Series(retArr, index=df.index)
1366 
1367 
1369  def _func(self, df):
1370  return self.dn2magdn2mag(df[self.colcol], self.fluxMag0fluxMag0)
1371 
1372 
1374  @property
1375  def columns(self):
1376  return [self.colcol, self.colFluxErrcolFluxErr]
1377 
1378  def _func(self, df):
1379  retArr = self.dn2MagErrdn2MagErr(df[self.colcol], df[self.colFluxErrcolFluxErr], self.fluxMag0fluxMag0, self.fluxMag0ErrfluxMag0Err)
1380  return pd.Series(retArr, index=df.index)
1381 
1382 
1384  """Base class for calibrating the specified instrument flux column using
1385  the local photometric calibration.
1386 
1387  Parameters
1388  ----------
1389  instFluxCol : `str`
1390  Name of the instrument flux column.
1391  instFluxErrCol : `str`
1392  Name of the assocated error columns for ``instFluxCol``.
1393  photoCalibCol : `str`
1394  Name of local calibration column.
1395  photoCalibErrCol : `str`
1396  Error associated with ``photoCalibCol``
1397 
1398  See also
1399  --------
1400  LocalPhotometry
1401  LocalNanojansky
1402  LocalNanojanskyErr
1403  LocalMagnitude
1404  LocalMagnitudeErr
1405  """
1406  logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1407 
1408  def __init__(self,
1409  instFluxCol,
1410  instFluxErrCol,
1411  photoCalibCol,
1412  photoCalibErrCol,
1413  **kwargs):
1414  self.instFluxColinstFluxCol = instFluxCol
1415  self.instFluxErrColinstFluxErrCol = instFluxErrCol
1416  self.photoCalibColphotoCalibCol = photoCalibCol
1417  self.photoCalibErrColphotoCalibErrCol = photoCalibErrCol
1418  super().__init__(**kwargs)
1419 
1420  def instFluxToNanojansky(self, instFlux, localCalib):
1421  """Convert instrument flux to nanojanskys.
1422 
1423  Parameters
1424  ----------
1425  instFlux : `numpy.ndarray` or `pandas.Series`
1426  Array of instrument flux measurements
1427  localCalib : `numpy.ndarray` or `pandas.Series`
1428  Array of local photometric calibration estimates.
1429 
1430  Returns
1431  -------
1432  calibFlux : `numpy.ndarray` or `pandas.Series`
1433  Array of calibrated flux measurements.
1434  """
1435  return instFlux * localCalib
1436 
1437  def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr):
1438  """Convert instrument flux to nanojanskys.
1439 
1440  Parameters
1441  ----------
1442  instFlux : `numpy.ndarray` or `pandas.Series`
1443  Array of instrument flux measurements
1444  instFluxErr : `numpy.ndarray` or `pandas.Series`
1445  Errors on associated ``instFlux`` values
1446  localCalib : `numpy.ndarray` or `pandas.Series`
1447  Array of local photometric calibration estimates.
1448  localCalibErr : `numpy.ndarray` or `pandas.Series`
1449  Errors on associated ``localCalib`` values
1450 
1451  Returns
1452  -------
1453  calibFluxErr : `numpy.ndarray` or `pandas.Series`
1454  Errors on calibrated flux measurements.
1455  """
1456  return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1457 
1458  def instFluxToMagnitude(self, instFlux, localCalib):
1459  """Convert instrument flux to nanojanskys.
1460 
1461  Parameters
1462  ----------
1463  instFlux : `numpy.ndarray` or `pandas.Series`
1464  Array of instrument flux measurements
1465  localCalib : `numpy.ndarray` or `pandas.Series`
1466  Array of local photometric calibration estimates.
1467 
1468  Returns
1469  -------
1470  calibMag : `numpy.ndarray` or `pandas.Series`
1471  Array of calibrated AB magnitudes.
1472  """
1473  return -2.5 * np.log10(self.instFluxToNanojanskyinstFluxToNanojansky(instFlux, localCalib)) + self.logNJanskyToABlogNJanskyToAB
1474 
1475  def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr):
1476  """Convert instrument flux err to nanojanskys.
1477 
1478  Parameters
1479  ----------
1480  instFlux : `numpy.ndarray` or `pandas.Series`
1481  Array of instrument flux measurements
1482  instFluxErr : `numpy.ndarray` or `pandas.Series`
1483  Errors on associated ``instFlux`` values
1484  localCalib : `numpy.ndarray` or `pandas.Series`
1485  Array of local photometric calibration estimates.
1486  localCalibErr : `numpy.ndarray` or `pandas.Series`
1487  Errors on associated ``localCalib`` values
1488 
1489  Returns
1490  -------
1491  calibMagErr: `numpy.ndarray` or `pandas.Series`
1492  Error on calibrated AB magnitudes.
1493  """
1494  err = self.instFluxErrToNanojanskyErrinstFluxErrToNanojanskyErr(instFlux, instFluxErr, localCalib, localCalibErr)
1495  return 2.5 / np.log(10) * err / self.instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1496 
1497 
1499  """Compute calibrated fluxes using the local calibration value.
1500 
1501  See also
1502  --------
1503  LocalNanojansky
1504  LocalNanojanskyErr
1505  LocalMagnitude
1506  LocalMagnitudeErr
1507  """
1508 
1509  @property
1510  def columns(self):
1511  return [self.instFluxColinstFluxCol, self.photoCalibColphotoCalibCol]
1512 
1513  @property
1514  def name(self):
1515  return f'flux_{self.instFluxCol}'
1516 
1517  def _func(self, df):
1518  return self.instFluxToNanojanskyinstFluxToNanojansky(df[self.instFluxColinstFluxCol], df[self.photoCalibColphotoCalibCol])
1519 
1520 
1522  """Compute calibrated flux errors using the local calibration value.
1523 
1524  See also
1525  --------
1526  LocalNanojansky
1527  LocalNanojanskyErr
1528  LocalMagnitude
1529  LocalMagnitudeErr
1530  """
1531 
1532  @property
1533  def columns(self):
1534  return [self.instFluxColinstFluxCol, self.instFluxErrColinstFluxErrCol,
1535  self.photoCalibColphotoCalibCol, self.photoCalibErrColphotoCalibErrCol]
1536 
1537  @property
1538  def name(self):
1539  return f'fluxErr_{self.instFluxCol}'
1540 
1541  def _func(self, df):
1542  return self.instFluxErrToNanojanskyErrinstFluxErrToNanojanskyErr(df[self.instFluxColinstFluxCol], df[self.instFluxErrColinstFluxErrCol],
1543  df[self.photoCalibColphotoCalibCol], df[self.photoCalibErrColphotoCalibErrCol])
1544 
1545 
1547  """Compute calibrated AB magnitudes using the local calibration value.
1548 
1549  See also
1550  --------
1551  LocalNanojansky
1552  LocalNanojanskyErr
1553  LocalMagnitude
1554  LocalMagnitudeErr
1555  """
1556 
1557  @property
1558  def columns(self):
1559  return [self.instFluxColinstFluxCol, self.photoCalibColphotoCalibCol]
1560 
1561  @property
1562  def name(self):
1563  return f'mag_{self.instFluxCol}'
1564 
1565  def _func(self, df):
1566  return self.instFluxToMagnitudeinstFluxToMagnitude(df[self.instFluxColinstFluxCol],
1567  df[self.photoCalibColphotoCalibCol])
1568 
1569 
1571  """Compute calibrated AB magnitude errors using the local calibration value.
1572 
1573  See also
1574  --------
1575  LocalNanojansky
1576  LocalNanojanskyErr
1577  LocalMagnitude
1578  LocalMagnitudeErr
1579  """
1580 
1581  @property
1582  def columns(self):
1583  return [self.instFluxColinstFluxCol, self.instFluxErrColinstFluxErrCol,
1584  self.photoCalibColphotoCalibCol, self.photoCalibErrColphotoCalibErrCol]
1585 
1586  @property
1587  def name(self):
1588  return f'magErr_{self.instFluxCol}'
1589 
1590  def _func(self, df):
1591  return self.instFluxErrToMagnitudeErrinstFluxErrToMagnitudeErr(df[self.instFluxColinstFluxCol],
1592  df[self.instFluxErrColinstFluxErrCol],
1593  df[self.photoCalibColphotoCalibCol],
1594  df[self.photoCalibErrColphotoCalibErrCol])
def multilevelColumns(self, parq, **kwargs)
Definition: functors.py:877
def __init__(self, col, filt2, filt1, **kwargs)
Definition: functors.py:848
def __init__(self, col, **kwargs)
Definition: functors.py:599
def __init__(self, funcs, **kwargs)
Definition: functors.py:381
def __call__(self, data, **kwargs)
Definition: functors.py:433
def from_file(cls, filename, **kwargs)
Definition: functors.py:505
def from_yaml(cls, translationDefinition, **kwargs)
Definition: functors.py:512
def renameCol(cls, col, renameRules)
Definition: functors.py:496
def multilevelColumns(self, data, **kwargs)
Definition: functors.py:419
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
Definition: functors.py:1207
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
Definition: functors.py:1247
def __init__(self, col, **kwargs)
Definition: functors.py:645
def __init__(self, expr, **kwargs)
Definition: functors.py:568
def __init__(self, **kwargs)
Definition: functors.py:673
def __call__(self, catalog, **kwargs)
Definition: functors.py:676
def __init__(self, colXX, colXY, colYY, **kwargs)
Definition: functors.py:1040
def __init__(self, colXX, colXY, colYY, **kwargs)
Definition: functors.py:1058
def __call__(self, data, dropna=False)
Definition: functors.py:318
def _func(self, df, dropna=True)
Definition: functors.py:261
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
Definition: functors.py:205
def _get_data_columnLevelNames(self, data, columnIndex=None)
Definition: functors.py:162
def difference(self, data1, data2, **kwargs)
Definition: functors.py:329
def __init__(self, filt=None, dataset=None, noDup=None)
Definition: functors.py:118
def _get_columnIndex(self, data)
Definition: functors.py:264
def _colsFromDict(self, colDict, columnIndex=None)
Definition: functors.py:184
def _get_data_columnLevels(self, data, columnIndex=None)
Definition: functors.py:138
def __call__(self, parq, dropna=False, **kwargs)
Definition: functors.py:897
def instFluxToNanojansky(self, instFlux, localCalib)
Definition: functors.py:1420
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
Definition: functors.py:1475
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
Definition: functors.py:1413
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
Definition: functors.py:1437
def instFluxToMagnitude(self, instFlux, localCalib)
Definition: functors.py:1458
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
Definition: functors.py:1098
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
Definition: functors.py:1105
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
Definition: functors.py:1134
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
Definition: functors.py:1160
def __init__(self, col1, col2, **kwargs)
Definition: functors.py:795
def __init__(self, *args, **kwargs)
Definition: functors.py:756
def __init__(self, col, calib=None, **kwargs)
Definition: functors.py:718
def dn2mag(self, dn, fluxMag0)
Definition: functors.py:1337
def dn2flux(self, dn, fluxMag0)
Definition: functors.py:1334
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
Definition: functors.py:1343
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
Definition: functors.py:1348
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
Definition: functors.py:1303
def __call__(self, catalog, **kwargs)
Definition: functors.py:663
def __init__(self, **kwargs)
Definition: functors.py:660
def __init__(self, colXX, colXY, colYY, **kwargs)
Definition: functors.py:1074
def mag_aware_eval(df, expr)
Definition: functors.py:533
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)
Definition: functors.py:15