lsst.pipe.tasks  21.0.0-33-g5ddf1805+888ad90e2c
functors.py
Go to the documentation of this file.
1 import yaml
2 import re
3 from itertools import product
4 
5 import pandas as pd
6 import numpy as np
7 import astropy.units as u
8 
9 from lsst.daf.persistence import doImport
10 from lsst.daf.butler import DeferredDatasetHandle
11 from .parquetTable import ParquetTable, MultilevelParquetTable
12 
13 
14 def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors',
15  typeKey='functor', name=None):
16  """Initialize an object defined in a dictionary
17 
18  The object needs to be importable as
19  f'{basePath}.{initDict[typeKey]}'
20  The positional and keyword arguments (if any) are contained in
21  "args" and "kwargs" entries in the dictionary, respectively.
22  This is used in `functors.CompositeFunctor.from_yaml` to initialize
23  a composite functor from a specification in a YAML file.
24 
25  Parameters
26  ----------
27  initDict : dictionary
28  Dictionary describing object's initialization. Must contain
29  an entry keyed by ``typeKey`` that is the name of the object,
30  relative to ``basePath``.
31  basePath : str
32  Path relative to module in which ``initDict[typeKey]`` is defined.
33  typeKey : str
34  Key of ``initDict`` that is the name of the object
35  (relative to `basePath`).
36  """
37  initDict = initDict.copy()
38  # TO DO: DM-21956 We should be able to define functors outside this module
39  pythonType = doImport(f'{basePath}.{initDict.pop(typeKey)}')
40  args = []
41  if 'args' in initDict:
42  args = initDict.pop('args')
43  if isinstance(args, str):
44  args = [args]
45  try:
46  element = pythonType(*args, **initDict)
47  except Exception as e:
48  message = f'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
49  raise type(e)(message, e.args)
50  return element
51 
52 
53 class Functor(object):
54  """Define and execute a calculation on a ParquetTable
55 
56  The `__call__` method accepts either a `ParquetTable` object or a
57  `DeferredDatasetHandle`, and returns the
58  result of the calculation as a single column. Each functor defines what
59  columns are needed for the calculation, and only these columns are read
60  from the `ParquetTable`.
61 
62  The action of `__call__` consists of two steps: first, loading the
63  necessary columns from disk into memory as a `pandas.DataFrame` object;
64  and second, performing the computation on this dataframe and returning the
65  result.
66 
67 
68  To define a new `Functor`, a subclass must define a `_func` method,
69  that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
70  In addition, it must define the following attributes
71 
72  * `_columns`: The columns necessary to perform the calculation
73  * `name`: A name appropriate for a figure axis label
74  * `shortname`: A name appropriate for use as a dictionary key
75 
76  On initialization, a `Functor` should declare what band (`filt` kwarg)
77  and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
78  applied to. This enables the `_get_data` method to extract the proper
79  columns from the parquet file. If not specified, the dataset will fall back
80  on the `_defaultDataset`attribute. If band is not specified and `dataset`
81  is anything other than `'ref'`, then an error will be raised when trying to
82  perform the calculation.
83 
84  As currently implemented, `Functor` is only set up to expect a
85  dataset of the format of the `deepCoadd_obj` dataset; that is, a
86  dataframe with a multi-level column index,
87  with the levels of the column index being `band`,
88  `dataset`, and `column`. This is defined in the `_columnLevels` attribute,
89  as well as being implicit in the role of the `filt` and `dataset` attributes
90  defined at initialization. In addition, the `_get_data` method that reads
91  the dataframe from the `ParquetTable` will return a dataframe with column
92  index levels defined by the `_dfLevels` attribute; by default, this is
93  `column`.
94 
95  The `_columnLevels` and `_dfLevels` attributes should generally not need to
96  be changed, unless `_func` needs columns from multiple filters or datasets
97  to do the calculation.
98  An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
99  which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
100  it gets to have those levels in the column index.
101 
102  Parameters
103  ----------
104  filt : str
105  Filter upon which to do the calculation
106 
107  dataset : str
108  Dataset upon which to do the calculation
109  (e.g., 'ref', 'meas', 'forced_src').
110 
111  """
112 
113  _defaultDataset = 'ref'
114  _columnLevels = ('band', 'dataset', 'column')
115  _dfLevels = ('column',)
116  _defaultNoDup = False
117 
118  def __init__(self, filt=None, dataset=None, noDup=None):
119  self.filtfilt = filt
120  self.datasetdataset = dataset if dataset is not None else self._defaultDataset_defaultDataset
121  self._noDup_noDup = noDup
122 
123  @property
124  def noDup(self):
125  if self._noDup_noDup is not None:
126  return self._noDup_noDup
127  else:
128  return self._defaultNoDup_defaultNoDup
129 
130  @property
131  def columns(self):
132  """Columns required to perform calculation
133  """
134  if not hasattr(self, '_columns'):
135  raise NotImplementedError('Must define columns property or _columns attribute')
136  return self._columns
137 
138  def _get_data_columnLevels(self, data, columnIndex=None):
139  """Gets the names of the column index levels
140 
141  This should only be called in the context of a multilevel table.
142  The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
143  and with the gen3 `DeferredDatasetHandle`.
144 
145  Parameters
146  ----------
147  data : `MultilevelParquetTable` or `DeferredDatasetHandle`
148 
149  columnnIndex (optional): pandas `Index` object
150  if not passed, then it is read from the `DeferredDatasetHandle`
151  """
152  if isinstance(data, DeferredDatasetHandle):
153  if columnIndex is None:
154  columnIndex = data.get(component="columns")
155  if columnIndex is not None:
156  return columnIndex.names
157  if isinstance(data, MultilevelParquetTable):
158  return data.columnLevels
159  else:
160  raise TypeError(f"Unknown type for data: {type(data)}!")
161 
162  def _get_data_columnLevelNames(self, data, columnIndex=None):
163  """Gets the content of each of the column levels for a multilevel table
164 
165  Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
166 
167  Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
168  """
169  if isinstance(data, DeferredDatasetHandle):
170  if columnIndex is None:
171  columnIndex = data.get(component="columns")
172  if columnIndex is not None:
173  columnLevels = columnIndex.names
174  columnLevelNames = {
175  level: list(np.unique(np.array([c for c in columnIndex])[:, i]))
176  for i, level in enumerate(columnLevels)
177  }
178  return columnLevelNames
179  if isinstance(data, MultilevelParquetTable):
180  return data.columnLevelNames
181  else:
182  raise TypeError(f"Unknown type for data: {type(data)}!")
183 
184  def _colsFromDict(self, colDict, columnIndex=None):
185  """Converts dictionary column specficiation to a list of columns
186 
187  This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
188  """
189  new_colDict = {}
190  columnLevels = self._get_data_columnLevels_get_data_columnLevels(None, columnIndex=columnIndex)
191 
192  for i, lev in enumerate(columnLevels):
193  if lev in colDict:
194  if isinstance(colDict[lev], str):
195  new_colDict[lev] = [colDict[lev]]
196  else:
197  new_colDict[lev] = colDict[lev]
198  else:
199  new_colDict[lev] = columnIndex.levels[i]
200 
201  levelCols = [new_colDict[lev] for lev in columnLevels]
202  cols = product(*levelCols)
203  return list(cols)
204 
205  def multilevelColumns(self, data, columnIndex=None, returnTuple=False):
206  """Returns columns needed by functor from multilevel dataset
207 
208  To access tables with multilevel column structure, the `MultilevelParquetTable`
209  or `DeferredDatasetHandle` need to be passed either a list of tuples or a
210  dictionary.
211 
212  Parameters
213  ----------
214  data : `MultilevelParquetTable` or `DeferredDatasetHandle`
215 
216  columnIndex (optional): pandas `Index` object
217  either passed or read in from `DeferredDatasetHandle`.
218 
219  `returnTuple` : bool
220  If true, then return a list of tuples rather than the column dictionary
221  specification. This is set to `True` by `CompositeFunctor` in order to be able to
222  combine columns from the various component functors.
223 
224  """
225  if isinstance(data, DeferredDatasetHandle) and columnIndex is None:
226  columnIndex = data.get(component="columns")
227 
228  # Confirm that the dataset has the column levels the functor is expecting it to have.
229  columnLevels = self._get_data_columnLevels_get_data_columnLevels(data, columnIndex)
230 
231  if not set(columnLevels) == set(self._columnLevels_columnLevels):
232  raise ValueError(
233  "ParquetTable does not have the expected column levels. "
234  f"Got {columnLevels}; expected {self._columnLevels}."
235  )
236 
237  columnDict = {'column': self.columnscolumns,
238  'dataset': self.datasetdataset}
239  if self.filtfilt is None:
240  columnLevelNames = self._get_data_columnLevelNames_get_data_columnLevelNames(data, columnIndex)
241  if "band" in columnLevels:
242  if self.datasetdataset == "ref":
243  columnDict["band"] = columnLevelNames["band"][0]
244  else:
245  raise ValueError(f"'filt' not set for functor {self.name}"
246  f"(dataset {self.dataset}) "
247  "and ParquetTable "
248  "contains multiple filters in column index. "
249  "Set 'filt' or set 'dataset' to 'ref'.")
250  else:
251  columnDict['band'] = self.filtfilt
252 
253  if isinstance(data, MultilevelParquetTable):
254  return data._colsFromDict(columnDict)
255  elif isinstance(data, DeferredDatasetHandle):
256  if returnTuple:
257  return self._colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
258  else:
259  return columnDict
260 
261  def _func(self, df, dropna=True):
262  raise NotImplementedError('Must define calculation on dataframe')
263 
264  def _get_columnIndex(self, data):
265  """Return columnIndex
266  """
267 
268  if isinstance(data, DeferredDatasetHandle):
269  return data.get(component="columns")
270  else:
271  return None
272 
273  def _get_data(self, data):
274  """Retrieve dataframe necessary for calculation.
275 
276  The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
277 
278  Returns dataframe upon which `self._func` can act.
279 
280  N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
281  """
282  if isinstance(data, pd.DataFrame):
283  return data
284 
285  # First thing to do: check to see if the data source has a multilevel column index or not.
286  columnIndex = self._get_columnIndex_get_columnIndex(data)
287  is_multiLevel = isinstance(data, MultilevelParquetTable) or isinstance(columnIndex, pd.MultiIndex)
288 
289  # Simple single-level parquet table, gen2
290  if isinstance(data, ParquetTable) and not is_multiLevel:
291  columns = self.columnscolumns
292  df = data.toDataFrame(columns=columns)
293  return df
294 
295  # Get proper multi-level columns specification for this functor
296  if is_multiLevel:
297  columns = self.multilevelColumnsmultilevelColumns(data, columnIndex=columnIndex)
298 
299  if isinstance(data, MultilevelParquetTable):
300  # Load in-memory dataframe with appropriate columns the gen2 way
301  df = data.toDataFrame(columns=columns, droplevels=False)
302  elif isinstance(data, DeferredDatasetHandle):
303  # Load in-memory dataframe with appropriate columns the gen3 way
304  df = data.get(parameters={"columns": columns})
305 
306  # Drop unnecessary column levels
307  df = self._setLevels_setLevels(df)
308  return df
309 
310  def _setLevels(self, df):
311  levelsToDrop = [n for n in df.columns.names if n not in self._dfLevels_dfLevels]
312  df.columns = df.columns.droplevel(levelsToDrop)
313  return df
314 
315  def _dropna(self, vals):
316  return vals.dropna()
317 
318  def __call__(self, data, dropna=False):
319  try:
320  df = self._get_data_get_data(data)
321  vals = self._func_func(df)
322  except Exception:
323  vals = self.failfail(df)
324  if dropna:
325  vals = self._dropna_dropna(vals)
326 
327  return vals
328 
329  def difference(self, data1, data2, **kwargs):
330  """Computes difference between functor called on two different ParquetTable objects
331  """
332  return self(data1, **kwargs) - self(data2, **kwargs)
333 
334  def fail(self, df):
335  return pd.Series(np.full(len(df), np.nan), index=df.index)
336 
337  @property
338  def name(self):
339  """Full name of functor (suitable for figure labels)
340  """
341  return NotImplementedError
342 
343  @property
344  def shortname(self):
345  """Short name of functor (suitable for column name/dict key)
346  """
347  return self.namename
348 
349 
351  """Perform multiple calculations at once on a catalog
352 
353  The role of a `CompositeFunctor` is to group together computations from
354  multiple functors. Instead of returning `pandas.Series` a
355  `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
356  being the keys of `funcDict`.
357 
358  The `columns` attribute of a `CompositeFunctor` is the union of all columns
359  in all the component functors.
360 
361  A `CompositeFunctor` does not use a `_func` method itself; rather,
362  when a `CompositeFunctor` is called, all its columns are loaded
363  at once, and the resulting dataframe is passed to the `_func` method of each component
364  functor. This has the advantage of only doing I/O (reading from parquet file) once,
365  and works because each individual `_func` method of each component functor does not
366  care if there are *extra* columns in the dataframe being passed; only that it must contain
367  *at least* the `columns` it expects.
368 
369  An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
370  file specifying a collection of functors.
371 
372  Parameters
373  ----------
374  funcs : `dict` or `list`
375  Dictionary or list of functors. If a list, then it will be converted
376  into a dictonary according to the `.shortname` attribute of each functor.
377 
378  """
379  dataset = None
380 
381  def __init__(self, funcs, **kwargs):
382 
383  if type(funcs) == dict:
384  self.funcDictfuncDict = funcs
385  else:
386  self.funcDictfuncDict = {f.shortname: f for f in funcs}
387 
388  self._filt_filt = None
389 
390  super().__init__(**kwargs)
391 
392  @property
393  def filt(self):
394  return self._filt_filt
395 
396  @filt.setter
397  def filt(self, filt):
398  if filt is not None:
399  for _, f in self.funcDictfuncDict.items():
400  f.filt = filt
401  self._filt_filt = filt
402 
403  def update(self, new):
404  if isinstance(new, dict):
405  self.funcDictfuncDict.update(new)
406  elif isinstance(new, CompositeFunctor):
407  self.funcDictfuncDict.update(new.funcDict)
408  else:
409  raise TypeError('Can only update with dictionary or CompositeFunctor.')
410 
411  # Make sure new functors have the same 'filt' set
412  if self.filtfiltfiltfiltfilt is not None:
413  self.filtfiltfiltfiltfilt = self.filtfiltfiltfiltfilt
414 
415  @property
416  def columns(self):
417  return list(set([x for y in [f.columns for f in self.funcDictfuncDict.values()] for x in y]))
418 
419  def multilevelColumns(self, data, **kwargs):
420  # Get the union of columns for all component functors. Note the need to have `returnTuple=True` here.
421  return list(
422  set(
423  [
424  x
425  for y in [
426  f.multilevelColumns(data, returnTuple=True, **kwargs) for f in self.funcDictfuncDict.values()
427  ]
428  for x in y
429  ]
430  )
431  )
432 
433  def __call__(self, data, **kwargs):
434  columnIndex = self._get_columnIndex_get_columnIndex(data)
435 
436  # First, determine whether data has a multilevel index (either gen2 or gen3)
437  is_multiLevel = isinstance(data, MultilevelParquetTable) or isinstance(columnIndex, pd.MultiIndex)
438 
439  # Simple single-level column index, gen2
440  if isinstance(data, ParquetTable) and not is_multiLevel:
441  columns = self.columnscolumnscolumns
442  df = data.toDataFrame(columns=columns)
443  valDict = {k: f._func(df) for k, f in self.funcDictfuncDict.items()}
444 
445  # Multilevel index, gen2 or gen3
446  if is_multiLevel:
447  columns = self.multilevelColumnsmultilevelColumnsmultilevelColumns(data, columnIndex=columnIndex)
448 
449  if isinstance(data, MultilevelParquetTable):
450  # Read data into memory the gen2 way
451  df = data.toDataFrame(columns=columns, droplevels=False)
452  elif isinstance(data, DeferredDatasetHandle):
453  # Read data into memory the gen3 way
454  df = data.get(parameters={"columns": columns})
455 
456  valDict = {}
457  for k, f in self.funcDictfuncDict.items():
458  try:
459  subdf = f._setLevels(
460  df[f.multilevelColumns(data, returnTuple=True, columnIndex=columnIndex)]
461  )
462  valDict[k] = f._func(subdf)
463  except Exception:
464  valDict[k] = f.fail(subdf)
465 
466  # non-multilevel, gen3 (TODO: this should work, but this case is not tested in test_functors.py)
467  elif isinstance(data, DeferredDatasetHandle):
468  columns = self.columnscolumnscolumns
469  df = data.get(parameters={"columns": columns})
470  valDict = {k: f._func(df) for k, f in self.funcDictfuncDict.items()}
471 
472  try:
473  valDf = pd.concat(valDict, axis=1)
474  except TypeError:
475  print([(k, type(v)) for k, v in valDict.items()])
476  raise
477 
478  if kwargs.get('dropna', False):
479  valDf = valDf.dropna(how='any')
480 
481  return valDf
482 
483  @classmethod
484  def renameCol(cls, col, renameRules):
485  if renameRules is None:
486  return col
487  for old, new in renameRules:
488  if col.startswith(old):
489  col = col.replace(old, new)
490  return col
491 
492  @classmethod
493  def from_file(cls, filename, **kwargs):
494  with open(filename) as f:
495  translationDefinition = yaml.safe_load(f)
496 
497  return cls.from_yamlfrom_yaml(translationDefinition, **kwargs)
498 
499  @classmethod
500  def from_yaml(cls, translationDefinition, **kwargs):
501  funcs = {}
502  for func, val in translationDefinition['funcs'].items():
503  funcs[func] = init_fromDict(val, name=func)
504 
505  if 'flag_rename_rules' in translationDefinition:
506  renameRules = translationDefinition['flag_rename_rules']
507  else:
508  renameRules = None
509 
510  if 'refFlags' in translationDefinition:
511  for flag in translationDefinition['refFlags']:
512  funcs[cls.renameColrenameCol(flag, renameRules)] = Column(flag, dataset='ref')
513 
514  if 'flags' in translationDefinition:
515  for flag in translationDefinition['flags']:
516  funcs[cls.renameColrenameCol(flag, renameRules)] = Column(flag, dataset='meas')
517 
518  return cls(funcs, **kwargs)
519 
520 
521 def mag_aware_eval(df, expr):
522  """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
523 
524  Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
525 
526  Parameters
527  ----------
528  df : pandas.DataFrame
529  Dataframe on which to evaluate expression.
530 
531  expr : str
532  Expression.
533  """
534  try:
535  expr_new = re.sub(r'mag\‍((\w+)\‍)', r'-2.5*log(\g<1>)/log(10)', expr)
536  val = df.eval(expr_new, truediv=True)
537  except Exception: # Should check what actually gets raised
538  expr_new = re.sub(r'mag\‍((\w+)\‍)', r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
539  val = df.eval(expr_new, truediv=True)
540  return val
541 
542 
544  """Arbitrary computation on a catalog
545 
546  Column names (and thus the columns to be loaded from catalog) are found
547  by finding all words and trying to ignore all "math-y" words.
548 
549  Parameters
550  ----------
551  expr : str
552  Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
553  """
554  _ignore_words = ('mag', 'sin', 'cos', 'exp', 'log', 'sqrt')
555 
556  def __init__(self, expr, **kwargs):
557  self.exprexpr = expr
558  super().__init__(**kwargs)
559 
560  @property
561  def name(self):
562  return self.exprexpr
563 
564  @property
565  def columns(self):
566  flux_cols = re.findall(r'mag\‍(\s*(\w+)\s*\‍)', self.exprexpr)
567 
568  cols = [c for c in re.findall(r'[a-zA-Z_]+', self.exprexpr) if c not in self._ignore_words_ignore_words]
569  not_a_col = []
570  for c in flux_cols:
571  if not re.search('_instFlux$', c):
572  cols.append(f'{c}_instFlux')
573  not_a_col.append(c)
574  else:
575  cols.append(c)
576 
577  return list(set([c for c in cols if c not in not_a_col]))
578 
579  def _func(self, df):
580  return mag_aware_eval(df, self.exprexpr)
581 
582 
584  """Get column with specified name
585  """
586 
587  def __init__(self, col, **kwargs):
588  self.colcol = col
589  super().__init__(**kwargs)
590 
591  @property
592  def name(self):
593  return self.colcol
594 
595  @property
596  def columns(self):
597  return [self.colcol]
598 
599  def _func(self, df):
600  return df[self.colcol]
601 
602 
603 class Index(Functor):
604  """Return the value of the index for each object
605  """
606 
607  columns = ['coord_ra'] # just a dummy; something has to be here
608  _defaultDataset = 'ref'
609  _defaultNoDup = True
610 
611  def _func(self, df):
612  return pd.Series(df.index, index=df.index)
613 
614 
616  col = 'id'
617  _allow_difference = False
618  _defaultNoDup = True
619 
620  def _func(self, df):
621  return pd.Series(df.index, index=df.index)
622 
623 
625  col = 'base_Footprint_nPix'
626 
627 
629  """Base class for coordinate column, in degrees
630  """
631  _radians = True
632 
633  def __init__(self, col, **kwargs):
634  super().__init__(col, **kwargs)
635 
636  def _func(self, df):
637  # Must not modify original column in case that column is used by another functor
638  output = df[self.colcol] * 180 / np.pi if self._radians_radians else df[self.colcol]
639  return output
640 
641 
643  """Right Ascension, in degrees
644  """
645  name = 'RA'
646  _defaultNoDup = True
647 
648  def __init__(self, **kwargs):
649  super().__init__('coord_ra', **kwargs)
650 
651  def __call__(self, catalog, **kwargs):
652  return super().__call__(catalog, **kwargs)
653 
654 
656  """Declination, in degrees
657  """
658  name = 'Dec'
659  _defaultNoDup = True
660 
661  def __init__(self, **kwargs):
662  super().__init__('coord_dec', **kwargs)
663 
664  def __call__(self, catalog, **kwargs):
665  return super().__call__(catalog, **kwargs)
666 
667 
668 def fluxName(col):
669  if not col.endswith('_instFlux'):
670  col += '_instFlux'
671  return col
672 
673 
674 def fluxErrName(col):
675  if not col.endswith('_instFluxErr'):
676  col += '_instFluxErr'
677  return col
678 
679 
680 class Mag(Functor):
681  """Compute calibrated magnitude
682 
683  Takes a `calib` argument, which returns the flux at mag=0
684  as `calib.getFluxMag0()`. If not provided, then the default
685  `fluxMag0` is 63095734448.0194, which is default for HSC.
686  This default should be removed in DM-21955
687 
688  This calculation hides warnings about invalid values and dividing by zero.
689 
690  As for all functors, a `dataset` and `filt` kwarg should be provided upon
691  initialization. Unlike the default `Functor`, however, the default dataset
692  for a `Mag` is `'meas'`, rather than `'ref'`.
693 
694  Parameters
695  ----------
696  col : `str`
697  Name of flux column from which to compute magnitude. Can be parseable
698  by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
699  `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
700  understand.
701  calib : `lsst.afw.image.calib.Calib` (optional)
702  Object that knows zero point.
703  """
704  _defaultDataset = 'meas'
705 
706  def __init__(self, col, calib=None, **kwargs):
707  self.colcol = fluxName(col)
708  self.calibcalib = calib
709  if calib is not None:
710  self.fluxMag0fluxMag0 = calib.getFluxMag0()[0]
711  else:
712  # TO DO: DM-21955 Replace hard coded photometic calibration values
713  self.fluxMag0fluxMag0 = 63095734448.0194
714 
715  super().__init__(**kwargs)
716 
717  @property
718  def columns(self):
719  return [self.colcol]
720 
721  def _func(self, df):
722  with np.warnings.catch_warnings():
723  np.warnings.filterwarnings('ignore', r'invalid value encountered')
724  np.warnings.filterwarnings('ignore', r'divide by zero')
725  return -2.5*np.log10(df[self.colcol] / self.fluxMag0fluxMag0)
726 
727  @property
728  def name(self):
729  return f'mag_{self.col}'
730 
731 
732 class MagErr(Mag):
733  """Compute calibrated magnitude uncertainty
734 
735  Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
736 
737  Parameters
738  col : `str`
739  Name of flux column
740  calib : `lsst.afw.image.calib.Calib` (optional)
741  Object that knows zero point.
742  """
743 
744  def __init__(self, *args, **kwargs):
745  super().__init__(*args, **kwargs)
746  if self.calibcalib is not None:
747  self.fluxMag0ErrfluxMag0Err = self.calibcalib.getFluxMag0()[1]
748  else:
749  self.fluxMag0ErrfluxMag0Err = 0.
750 
751  @property
752  def columns(self):
753  return [self.colcol, self.colcol + 'Err']
754 
755  def _func(self, df):
756  with np.warnings.catch_warnings():
757  np.warnings.filterwarnings('ignore', r'invalid value encountered')
758  np.warnings.filterwarnings('ignore', r'divide by zero')
759  fluxCol, fluxErrCol = self.columnscolumnscolumnscolumns
760  x = df[fluxErrCol] / df[fluxCol]
761  y = self.fluxMag0ErrfluxMag0Err / self.fluxMag0fluxMag0
762  magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
763  return magErr
764 
765  @property
766  def name(self):
767  return super().name + '_err'
768 
769 
771  """
772  """
773 
774  def _func(self, df):
775  return (df[self.colcol] / self.fluxMag0fluxMag0) * 1e9
776 
777 
779  _defaultDataset = 'meas'
780 
781  """Functor to calculate magnitude difference"""
782 
783  def __init__(self, col1, col2, **kwargs):
784  self.col1col1 = fluxName(col1)
785  self.col2col2 = fluxName(col2)
786  super().__init__(**kwargs)
787 
788  @property
789  def columns(self):
790  return [self.col1col1, self.col2col2]
791 
792  def _func(self, df):
793  with np.warnings.catch_warnings():
794  np.warnings.filterwarnings('ignore', r'invalid value encountered')
795  np.warnings.filterwarnings('ignore', r'divide by zero')
796  return -2.5*np.log10(df[self.col1col1]/df[self.col2col2])
797 
798  @property
799  def name(self):
800  return f'(mag_{self.col1} - mag_{self.col2})'
801 
802  @property
803  def shortname(self):
804  return f'magDiff_{self.col1}_{self.col2}'
805 
806 
807 class Color(Functor):
808  """Compute the color between two filters
809 
810  Computes color by initializing two different `Mag`
811  functors based on the `col` and filters provided, and
812  then returning the difference.
813 
814  This is enabled by the `_func` expecting a dataframe with a
815  multilevel column index, with both `'band'` and `'column'`,
816  instead of just `'column'`, which is the `Functor` default.
817  This is controlled by the `_dfLevels` attribute.
818 
819  Also of note, the default dataset for `Color` is `forced_src'`,
820  whereas for `Mag` it is `'meas'`.
821 
822  Parameters
823  ----------
824  col : str
825  Name of flux column from which to compute; same as would be passed to
826  `lsst.pipe.tasks.functors.Mag`.
827 
828  filt2, filt1 : str
829  Filters from which to compute magnitude difference.
830  Color computed is `Mag(filt2) - Mag(filt1)`.
831  """
832  _defaultDataset = 'forced_src'
833  _dfLevels = ('band', 'column')
834  _defaultNoDup = True
835 
836  def __init__(self, col, filt2, filt1, **kwargs):
837  self.colcol = fluxName(col)
838  if filt2 == filt1:
839  raise RuntimeError("Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
840  self.filt2filt2 = filt2
841  self.filt1filt1 = filt1
842 
843  self.mag2mag2 = Mag(col, filt=filt2, **kwargs)
844  self.mag1mag1 = Mag(col, filt=filt1, **kwargs)
845 
846  super().__init__(**kwargs)
847 
848  @property
849  def filt(self):
850  return None
851 
852  @filt.setter
853  def filt(self, filt):
854  pass
855 
856  def _func(self, df):
857  mag2 = self.mag2._func(df[self.filt2])
858  mag1 = self.mag1._func(df[self.filt1])
859  return mag2 - mag1
860 
861  @property
862  def columns(self):
863  return [self.mag1mag1.col, self.mag2mag2.col]
864 
865  def multilevelColumns(self, parq, **kwargs):
866  return [(self.datasetdataset, self.filt1filt1, self.colcol), (self.datasetdataset, self.filt2filt2, self.colcol)]
867 
868  @property
869  def name(self):
870  return f'{self.filt2} - {self.filt1} ({self.col})'
871 
872  @property
873  def shortname(self):
874  return f"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
875 
876 
878  """Main function of this subclass is to override the dropna=True
879  """
880  _null_label = 'null'
881  _allow_difference = False
882  name = 'label'
883  _force_str = False
884 
885  def __call__(self, parq, dropna=False, **kwargs):
886  return super().__call__(parq, dropna=False, **kwargs)
887 
888 
890  _columns = ["base_ClassificationExtendedness_value"]
891  _column = "base_ClassificationExtendedness_value"
892 
893  def _func(self, df):
894  x = df[self._columns_columns][self._column_column]
895  mask = x.isnull()
896  test = (x < 0.5).astype(int)
897  test = test.mask(mask, 2)
898 
899  # TODO: DM-21954 Look into veracity of inline comment below
900  # are these backwards?
901  categories = ['galaxy', 'star', self._null_label_null_label]
902  label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
903  index=x.index, name='label')
904  if self._force_str_force_str:
905  label = label.astype(str)
906  return label
907 
908 
910  _columns = ['numStarFlags']
911  labels = {"star": 0, "maybe": 1, "notStar": 2}
912 
913  def _func(self, df):
914  x = df[self._columns_columns][self._columns_columns[0]]
915 
916  # Number of filters
917  n = len(x.unique()) - 1
918 
919  labels = ['noStar', 'maybe', 'star']
920  label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
921  index=x.index, name='label')
922 
923  if self._force_str_force_str:
924  label = label.astype(str)
925 
926  return label
927 
928 
930  name = 'Deconvolved Moments'
931  shortname = 'deconvolvedMoments'
932  _columns = ("ext_shapeHSM_HsmSourceMoments_xx",
933  "ext_shapeHSM_HsmSourceMoments_yy",
934  "base_SdssShape_xx", "base_SdssShape_yy",
935  "ext_shapeHSM_HsmPsfMoments_xx",
936  "ext_shapeHSM_HsmPsfMoments_yy")
937 
938  def _func(self, df):
939  """Calculate deconvolved moments"""
940  if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns: # _xx added by tdm
941  hsm = df["ext_shapeHSM_HsmSourceMoments_xx"] + df["ext_shapeHSM_HsmSourceMoments_yy"]
942  else:
943  hsm = np.ones(len(df))*np.nan
944  sdss = df["base_SdssShape_xx"] + df["base_SdssShape_yy"]
945  if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
946  psf = df["ext_shapeHSM_HsmPsfMoments_xx"] + df["ext_shapeHSM_HsmPsfMoments_yy"]
947  else:
948  # LSST does not have shape.sdss.psf. Could instead add base_PsfShape to catalog using
949  # exposure.getPsf().computeShape(s.getCentroid()).getIxx()
950  # raise TaskError("No psf shape parameter found in catalog")
951  raise RuntimeError('No psf shape parameter found in catalog')
952 
953  return hsm.where(np.isfinite(hsm), sdss) - psf
954 
955 
957  """Functor to calculate SDSS trace radius size for sources"""
958  name = "SDSS Trace Size"
959  shortname = 'sdssTrace'
960  _columns = ("base_SdssShape_xx", "base_SdssShape_yy")
961 
962  def _func(self, df):
963  srcSize = np.sqrt(0.5*(df["base_SdssShape_xx"] + df["base_SdssShape_yy"]))
964  return srcSize
965 
966 
968  """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
969  name = "PSF - SDSS Trace Size"
970  shortname = 'psf_sdssTrace'
971  _columns = ("base_SdssShape_xx", "base_SdssShape_yy",
972  "base_SdssShape_psf_xx", "base_SdssShape_psf_yy")
973 
974  def _func(self, df):
975  srcSize = np.sqrt(0.5*(df["base_SdssShape_xx"] + df["base_SdssShape_yy"]))
976  psfSize = np.sqrt(0.5*(df["base_SdssShape_psf_xx"] + df["base_SdssShape_psf_yy"]))
977  sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
978  return sizeDiff
979 
980 
982  """Functor to calculate HSM trace radius size for sources"""
983  name = 'HSM Trace Size'
984  shortname = 'hsmTrace'
985  _columns = ("ext_shapeHSM_HsmSourceMoments_xx",
986  "ext_shapeHSM_HsmSourceMoments_yy")
987 
988  def _func(self, df):
989  srcSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmSourceMoments_xx"]
990  + df["ext_shapeHSM_HsmSourceMoments_yy"]))
991  return srcSize
992 
993 
995  """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
996  name = 'PSF - HSM Trace Size'
997  shortname = 'psf_HsmTrace'
998  _columns = ("ext_shapeHSM_HsmSourceMoments_xx",
999  "ext_shapeHSM_HsmSourceMoments_yy",
1000  "ext_shapeHSM_HsmPsfMoments_xx",
1001  "ext_shapeHSM_HsmPsfMoments_yy")
1002 
1003  def _func(self, df):
1004  srcSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmSourceMoments_xx"]
1005  + df["ext_shapeHSM_HsmSourceMoments_yy"]))
1006  psfSize = np.sqrt(0.5*(df["ext_shapeHSM_HsmPsfMoments_xx"]
1007  + df["ext_shapeHSM_HsmPsfMoments_yy"]))
1008  sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1009  return sizeDiff
1010 
1011 
1013  name = 'HSM Psf FWHM'
1014  _columns = ('ext_shapeHSM_HsmPsfMoments_xx', 'ext_shapeHSM_HsmPsfMoments_yy')
1015  # TODO: DM-21403 pixel scale should be computed from the CD matrix or transform matrix
1016  pixelScale = 0.168
1017  SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1018 
1019  def _func(self, df):
1020  return self.pixelScalepixelScale*self.SIGMA2FWHMSIGMA2FWHM*np.sqrt(
1021  0.5*(df['ext_shapeHSM_HsmPsfMoments_xx'] + df['ext_shapeHSM_HsmPsfMoments_yy']))
1022 
1023 
1024 class E1(Functor):
1025  name = "Distortion Ellipticity (e1)"
1026  shortname = "Distortion"
1027 
1028  def __init__(self, colXX, colXY, colYY, **kwargs):
1029  self.colXXcolXX = colXX
1030  self.colXYcolXY = colXY
1031  self.colYYcolYY = colYY
1032  self._columns_columns = [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1033  super().__init__(**kwargs)
1034 
1035  @property
1036  def columns(self):
1037  return [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1038 
1039  def _func(self, df):
1040  return df[self.colXXcolXX] - df[self.colYYcolYY] / (df[self.colXXcolXX] + df[self.colYYcolYY])
1041 
1042 
1043 class E2(Functor):
1044  name = "Ellipticity e2"
1045 
1046  def __init__(self, colXX, colXY, colYY, **kwargs):
1047  self.colXXcolXX = colXX
1048  self.colXYcolXY = colXY
1049  self.colYYcolYY = colYY
1050  super().__init__(**kwargs)
1051 
1052  @property
1053  def columns(self):
1054  return [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1055 
1056  def _func(self, df):
1057  return 2*df[self.colXYcolXY] / (df[self.colXXcolXX] + df[self.colYYcolYY])
1058 
1059 
1061 
1062  def __init__(self, colXX, colXY, colYY, **kwargs):
1063  self.colXXcolXX = colXX
1064  self.colXYcolXY = colXY
1065  self.colYYcolYY = colYY
1066  super().__init__(**kwargs)
1067 
1068  @property
1069  def columns(self):
1070  return [self.colXXcolXX, self.colXYcolXY, self.colYYcolYY]
1071 
1072  def _func(self, df):
1073  return (df[self.colXXcolXX]*df[self.colYYcolYY] - df[self.colXYcolXY]**2)**0.25
1074 
1075 
1077  """Computations using the stored localWcs.
1078  """
1079  name = "LocalWcsOperations"
1080 
1081  def __init__(self,
1082  colCD_1_1,
1083  colCD_1_2,
1084  colCD_2_1,
1085  colCD_2_2,
1086  **kwargs):
1087  self.colCD_1_1colCD_1_1 = colCD_1_1
1088  self.colCD_1_2colCD_1_2 = colCD_1_2
1089  self.colCD_2_1colCD_2_1 = colCD_2_1
1090  self.colCD_2_2colCD_2_2 = colCD_2_2
1091  super().__init__(**kwargs)
1092 
1093  def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22):
1094  """Compute the distance on the sphere from x2, y1 to x1, y1.
1095 
1096  Parameters
1097  ----------
1098  x : `pandas.Series`
1099  X pixel coordinate.
1100  y : `pandas.Series`
1101  Y pixel coordinate.
1102  cd11 : `pandas.Series`
1103  [1, 1] element of the local Wcs affine transform.
1104  cd11 : `pandas.Series`
1105  [1, 1] element of the local Wcs affine transform.
1106  cd12 : `pandas.Series`
1107  [1, 2] element of the local Wcs affine transform.
1108  cd21 : `pandas.Series`
1109  [2, 1] element of the local Wcs affine transform.
1110  cd22 : `pandas.Series`
1111  [2, 2] element of the local Wcs affine transform.
1112 
1113  Returns
1114  -------
1115  raDecTuple : tuple
1116  RA and dec conversion of x and y given the local Wcs. Returned
1117  units are in radians.
1118 
1119  """
1120  return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1121 
1122  def computeSkySeperation(self, ra1, dec1, ra2, dec2):
1123  """Compute the local pixel scale conversion.
1124 
1125  Parameters
1126  ----------
1127  ra1 : `pandas.Series`
1128  Ra of the first coordinate in radians.
1129  dec1 : `pandas.Series`
1130  Dec of the first coordinate in radians.
1131  ra2 : `pandas.Series`
1132  Ra of the second coordinate in radians.
1133  dec2 : `pandas.Series`
1134  Dec of the second coordinate in radians.
1135 
1136  Returns
1137  -------
1138  dist : `pandas.Series`
1139  Distance on the sphere in radians.
1140  """
1141  deltaDec = dec2 - dec1
1142  deltaRa = ra2 - ra1
1143  return 2 * np.arcsin(
1144  np.sqrt(
1145  np.sin(deltaDec / 2) ** 2
1146  + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1147 
1148  def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22):
1149  """Compute the distance on the sphere from x2, y1 to x1, y1.
1150 
1151  Parameters
1152  ----------
1153  x1 : `pandas.Series`
1154  X pixel coordinate.
1155  y1 : `pandas.Series`
1156  Y pixel coordinate.
1157  x2 : `pandas.Series`
1158  X pixel coordinate.
1159  y2 : `pandas.Series`
1160  Y pixel coordinate.
1161  cd11 : `pandas.Series`
1162  [1, 1] element of the local Wcs affine transform.
1163  cd11 : `pandas.Series`
1164  [1, 1] element of the local Wcs affine transform.
1165  cd12 : `pandas.Series`
1166  [1, 2] element of the local Wcs affine transform.
1167  cd21 : `pandas.Series`
1168  [2, 1] element of the local Wcs affine transform.
1169  cd22 : `pandas.Series`
1170  [2, 2] element of the local Wcs affine transform.
1171 
1172  Returns
1173  -------
1174  Distance : `pandas.Series`
1175  Arcseconds per pixel at the location of the local WC
1176  """
1177  ra1, dec1 = self.computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1178  ra2, dec2 = self.computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1179  # Great circle distance for small separations.
1180  return self.computeSkySeperationcomputeSkySeperation(ra1, dec1, ra2, dec2)
1181 
1182 
1184  """Compute the local pixel scale from the stored CDMatrix.
1185  """
1186  name = "PixelScale"
1187 
1188  @property
1189  def columns(self):
1190  return [self.colCD_1_1colCD_1_1,
1191  self.colCD_1_2colCD_1_2,
1192  self.colCD_2_1colCD_2_1,
1193  self.colCD_2_2colCD_2_2]
1194 
1195  def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22):
1196  """Compute the local pixel to scale conversion in arcseconds.
1197 
1198  Parameters
1199  ----------
1200  cd11 : `pandas.Series`
1201  [1, 1] element of the local Wcs affine transform in radians.
1202  cd11 : `pandas.Series`
1203  [1, 1] element of the local Wcs affine transform in radians.
1204  cd12 : `pandas.Series`
1205  [1, 2] element of the local Wcs affine transform in radians.
1206  cd21 : `pandas.Series`
1207  [2, 1] element of the local Wcs affine transform in radians.
1208  cd22 : `pandas.Series`
1209  [2, 2] element of the local Wcs affine transform in radians.
1210 
1211  Returns
1212  -------
1213  pixScale : `pandas.Series`
1214  Arcseconds per pixel at the location of the local WC
1215  """
1216  return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1217 
1218  def _func(self, df):
1219  return self.pixelScaleArcsecondspixelScaleArcseconds(df[self.colCD_1_1colCD_1_1],
1220  df[self.colCD_1_2colCD_1_2],
1221  df[self.colCD_2_1colCD_2_1],
1222  df[self.colCD_2_2colCD_2_2])
1223 
1224 
1226  """Convert a value in units pixels to units arcseconds.
1227  """
1228 
1229  def __init__(self,
1230  col,
1231  colCD_1_1,
1232  colCD_1_2,
1233  colCD_2_1,
1234  colCD_2_2,
1235  **kwargs):
1236  self.colcol = col
1237  super().__init__(colCD_1_1,
1238  colCD_1_2,
1239  colCD_2_1,
1240  colCD_2_2,
1241  **kwargs)
1242 
1243  @property
1244  def name(self):
1245  return f"{self.col}_asArcseconds"
1246 
1247  @property
1248  def columns(self):
1249  return [self.colcol,
1250  self.colCD_1_1colCD_1_1,
1251  self.colCD_1_2colCD_1_2,
1252  self.colCD_2_1colCD_2_1,
1253  self.colCD_2_2colCD_2_2]
1254 
1255  def _func(self, df):
1256  return df[self.colcol] * self.pixelScaleArcsecondspixelScaleArcseconds(df[self.colCD_1_1colCD_1_1],
1257  df[self.colCD_1_2colCD_1_2],
1258  df[self.colCD_2_1colCD_2_1],
1259  df[self.colCD_2_2colCD_2_2])
1260 
1261 
1263  name = 'Reference Band'
1264  shortname = 'refBand'
1265 
1266  @property
1267  def columns(self):
1268  return ["merge_measurement_i",
1269  "merge_measurement_r",
1270  "merge_measurement_z",
1271  "merge_measurement_y",
1272  "merge_measurement_g"]
1273 
1274  def _func(self, df):
1275  def getFilterAliasName(row):
1276  # get column name with the max value (True > False)
1277  colName = row.idxmax()
1278  return colName.replace('merge_measurement_', '')
1279 
1280  return df[self.columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1281 
1282 
1284  # AB to NanoJansky (3631 Jansky)
1285  AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1286  LOG_AB_FLUX_SCALE = 12.56
1287  FIVE_OVER_2LOG10 = 1.085736204758129569
1288  # TO DO: DM-21955 Replace hard coded photometic calibration values
1289  COADD_ZP = 27
1290 
1291  def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1292  self.vhypotvhypot = np.vectorize(self.hypothypot)
1293  self.colcol = colFlux
1294  self.colFluxErrcolFluxErr = colFluxErr
1295 
1296  self.calibcalib = calib
1297  if calib is not None:
1298  self.fluxMag0fluxMag0, self.fluxMag0ErrfluxMag0Err = calib.getFluxMag0()
1299  else:
1300  self.fluxMag0fluxMag0 = 1./np.power(10, -0.4*self.COADD_ZPCOADD_ZP)
1301  self.fluxMag0ErrfluxMag0Err = 0.
1302 
1303  super().__init__(**kwargs)
1304 
1305  @property
1306  def columns(self):
1307  return [self.colcol]
1308 
1309  @property
1310  def name(self):
1311  return f'mag_{self.col}'
1312 
1313  @classmethod
1314  def hypot(cls, a, b):
1315  if np.abs(a) < np.abs(b):
1316  a, b = b, a
1317  if a == 0.:
1318  return 0.
1319  q = b/a
1320  return np.abs(a) * np.sqrt(1. + q*q)
1321 
1322  def dn2flux(self, dn, fluxMag0):
1323  return self.AB_FLUX_SCALEAB_FLUX_SCALE * dn / fluxMag0
1324 
1325  def dn2mag(self, dn, fluxMag0):
1326  with np.warnings.catch_warnings():
1327  np.warnings.filterwarnings('ignore', r'invalid value encountered')
1328  np.warnings.filterwarnings('ignore', r'divide by zero')
1329  return -2.5 * np.log10(dn/fluxMag0)
1330 
1331  def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err):
1332  retVal = self.vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1333  retVal *= self.AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1334  return retVal
1335 
1336  def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err):
1337  retVal = self.dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.dn2fluxdn2flux(dn, fluxMag0)
1338  return self.FIVE_OVER_2LOG10FIVE_OVER_2LOG10 * retVal
1339 
1340 
1342  def _func(self, df):
1343  return self.dn2fluxdn2flux(df[self.colcol], self.fluxMag0fluxMag0)
1344 
1345 
1347  @property
1348  def columns(self):
1349  return [self.colcol, self.colFluxErrcolFluxErr]
1350 
1351  def _func(self, df):
1352  retArr = self.dn2fluxErrdn2fluxErr(df[self.colcol], df[self.colFluxErrcolFluxErr], self.fluxMag0fluxMag0, self.fluxMag0ErrfluxMag0Err)
1353  return pd.Series(retArr, index=df.index)
1354 
1355 
1357  def _func(self, df):
1358  return self.dn2magdn2mag(df[self.colcol], self.fluxMag0fluxMag0)
1359 
1360 
1362  @property
1363  def columns(self):
1364  return [self.colcol, self.colFluxErrcolFluxErr]
1365 
1366  def _func(self, df):
1367  retArr = self.dn2MagErrdn2MagErr(df[self.colcol], df[self.colFluxErrcolFluxErr], self.fluxMag0fluxMag0, self.fluxMag0ErrfluxMag0Err)
1368  return pd.Series(retArr, index=df.index)
1369 
1370 
1372  """Base class for calibrating the specified instrument flux column using
1373  the local photometric calibration.
1374 
1375  Parameters
1376  ----------
1377  instFluxCol : `str`
1378  Name of the instrument flux column.
1379  instFluxErrCol : `str`
1380  Name of the assocated error columns for ``instFluxCol``.
1381  photoCalibCol : `str`
1382  Name of local calibration column.
1383  photoCalibErrCol : `str`
1384  Error associated with ``photoCalibCol``
1385 
1386  See also
1387  --------
1388  LocalPhotometry
1389  LocalNanojansky
1390  LocalNanojanskyErr
1391  LocalMagnitude
1392  LocalMagnitudeErr
1393  """
1394  logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1395 
1396  def __init__(self,
1397  instFluxCol,
1398  instFluxErrCol,
1399  photoCalibCol,
1400  photoCalibErrCol,
1401  **kwargs):
1402  self.instFluxColinstFluxCol = instFluxCol
1403  self.instFluxErrColinstFluxErrCol = instFluxErrCol
1404  self.photoCalibColphotoCalibCol = photoCalibCol
1405  self.photoCalibErrColphotoCalibErrCol = photoCalibErrCol
1406  super().__init__(**kwargs)
1407 
1408  def instFluxToNanojansky(self, instFlux, localCalib):
1409  """Convert instrument flux to nanojanskys.
1410 
1411  Parameters
1412  ----------
1413  instFlux : `numpy.ndarray` or `pandas.Series`
1414  Array of instrument flux measurements
1415  localCalib : `numpy.ndarray` or `pandas.Series`
1416  Array of local photometric calibration estimates.
1417 
1418  Returns
1419  -------
1420  calibFlux : `numpy.ndarray` or `pandas.Series`
1421  Array of calibrated flux measurements.
1422  """
1423  return instFlux * localCalib
1424 
1425  def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr):
1426  """Convert instrument flux to nanojanskys.
1427 
1428  Parameters
1429  ----------
1430  instFlux : `numpy.ndarray` or `pandas.Series`
1431  Array of instrument flux measurements
1432  instFluxErr : `numpy.ndarray` or `pandas.Series`
1433  Errors on associated ``instFlux`` values
1434  localCalib : `numpy.ndarray` or `pandas.Series`
1435  Array of local photometric calibration estimates.
1436  localCalibErr : `numpy.ndarray` or `pandas.Series`
1437  Errors on associated ``localCalib`` values
1438 
1439  Returns
1440  -------
1441  calibFluxErr : `numpy.ndarray` or `pandas.Series`
1442  Errors on calibrated flux measurements.
1443  """
1444  return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1445 
1446  def instFluxToMagnitude(self, instFlux, localCalib):
1447  """Convert instrument flux to nanojanskys.
1448 
1449  Parameters
1450  ----------
1451  instFlux : `numpy.ndarray` or `pandas.Series`
1452  Array of instrument flux measurements
1453  localCalib : `numpy.ndarray` or `pandas.Series`
1454  Array of local photometric calibration estimates.
1455 
1456  Returns
1457  -------
1458  calibMag : `numpy.ndarray` or `pandas.Series`
1459  Array of calibrated AB magnitudes.
1460  """
1461  return -2.5 * np.log10(self.instFluxToNanojanskyinstFluxToNanojansky(instFlux, localCalib)) + self.logNJanskyToABlogNJanskyToAB
1462 
1463  def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr):
1464  """Convert instrument flux err to nanojanskys.
1465 
1466  Parameters
1467  ----------
1468  instFlux : `numpy.ndarray` or `pandas.Series`
1469  Array of instrument flux measurements
1470  instFluxErr : `numpy.ndarray` or `pandas.Series`
1471  Errors on associated ``instFlux`` values
1472  localCalib : `numpy.ndarray` or `pandas.Series`
1473  Array of local photometric calibration estimates.
1474  localCalibErr : `numpy.ndarray` or `pandas.Series`
1475  Errors on associated ``localCalib`` values
1476 
1477  Returns
1478  -------
1479  calibMagErr: `numpy.ndarray` or `pandas.Series`
1480  Error on calibrated AB magnitudes.
1481  """
1482  err = self.instFluxErrToNanojanskyErrinstFluxErrToNanojanskyErr(instFlux, instFluxErr, localCalib, localCalibErr)
1483  return 2.5 / np.log(10) * err / self.instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1484 
1485 
1487  """Compute calibrated fluxes using the local calibration value.
1488 
1489  See also
1490  --------
1491  LocalNanojansky
1492  LocalNanojanskyErr
1493  LocalMagnitude
1494  LocalMagnitudeErr
1495  """
1496 
1497  @property
1498  def columns(self):
1499  return [self.instFluxColinstFluxCol, self.photoCalibColphotoCalibCol]
1500 
1501  @property
1502  def name(self):
1503  return f'flux_{self.instFluxCol}'
1504 
1505  def _func(self, df):
1506  return self.instFluxToNanojanskyinstFluxToNanojansky(df[self.instFluxColinstFluxCol], df[self.photoCalibColphotoCalibCol])
1507 
1508 
1510  """Compute calibrated flux errors using the local calibration value.
1511 
1512  See also
1513  --------
1514  LocalNanojansky
1515  LocalNanojanskyErr
1516  LocalMagnitude
1517  LocalMagnitudeErr
1518  """
1519 
1520  @property
1521  def columns(self):
1522  return [self.instFluxColinstFluxCol, self.instFluxErrColinstFluxErrCol,
1523  self.photoCalibColphotoCalibCol, self.photoCalibErrColphotoCalibErrCol]
1524 
1525  @property
1526  def name(self):
1527  return f'fluxErr_{self.instFluxCol}'
1528 
1529  def _func(self, df):
1530  return self.instFluxErrToNanojanskyErrinstFluxErrToNanojanskyErr(df[self.instFluxColinstFluxCol], df[self.instFluxErrColinstFluxErrCol],
1531  df[self.photoCalibColphotoCalibCol], df[self.photoCalibErrColphotoCalibErrCol])
1532 
1533 
1535  """Compute calibrated AB magnitudes using the local calibration value.
1536 
1537  See also
1538  --------
1539  LocalNanojansky
1540  LocalNanojanskyErr
1541  LocalMagnitude
1542  LocalMagnitudeErr
1543  """
1544 
1545  @property
1546  def columns(self):
1547  return [self.instFluxColinstFluxCol, self.photoCalibColphotoCalibCol]
1548 
1549  @property
1550  def name(self):
1551  return f'mag_{self.instFluxCol}'
1552 
1553  def _func(self, df):
1554  return self.instFluxToMagnitudeinstFluxToMagnitude(df[self.instFluxColinstFluxCol],
1555  df[self.photoCalibColphotoCalibCol])
1556 
1557 
1559  """Compute calibrated AB magnitude errors using the local calibration value.
1560 
1561  See also
1562  --------
1563  LocalNanojansky
1564  LocalNanojanskyErr
1565  LocalMagnitude
1566  LocalMagnitudeErr
1567  """
1568 
1569  @property
1570  def columns(self):
1571  return [self.instFluxColinstFluxCol, self.instFluxErrColinstFluxErrCol,
1572  self.photoCalibColphotoCalibCol, self.photoCalibErrColphotoCalibErrCol]
1573 
1574  @property
1575  def name(self):
1576  return f'magErr_{self.instFluxCol}'
1577 
1578  def _func(self, df):
1579  return self.instFluxErrToMagnitudeErrinstFluxErrToMagnitudeErr(df[self.instFluxColinstFluxCol],
1580  df[self.instFluxErrColinstFluxErrCol],
1581  df[self.photoCalibColphotoCalibCol],
1582  df[self.photoCalibErrColphotoCalibErrCol])
def multilevelColumns(self, parq, **kwargs)
Definition: functors.py:865
def __init__(self, col, filt2, filt1, **kwargs)
Definition: functors.py:836
def __init__(self, col, **kwargs)
Definition: functors.py:587
def __init__(self, funcs, **kwargs)
Definition: functors.py:381
def __call__(self, data, **kwargs)
Definition: functors.py:433
def from_file(cls, filename, **kwargs)
Definition: functors.py:493
def from_yaml(cls, translationDefinition, **kwargs)
Definition: functors.py:500
def renameCol(cls, col, renameRules)
Definition: functors.py:484
def multilevelColumns(self, data, **kwargs)
Definition: functors.py:419
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
Definition: functors.py:1195
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
Definition: functors.py:1235
def __init__(self, col, **kwargs)
Definition: functors.py:633
def __init__(self, expr, **kwargs)
Definition: functors.py:556
def __init__(self, **kwargs)
Definition: functors.py:661
def __call__(self, catalog, **kwargs)
Definition: functors.py:664
def __init__(self, colXX, colXY, colYY, **kwargs)
Definition: functors.py:1028
def __init__(self, colXX, colXY, colYY, **kwargs)
Definition: functors.py:1046
def __call__(self, data, dropna=False)
Definition: functors.py:318
def _func(self, df, dropna=True)
Definition: functors.py:261
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
Definition: functors.py:205
def _get_data_columnLevelNames(self, data, columnIndex=None)
Definition: functors.py:162
def difference(self, data1, data2, **kwargs)
Definition: functors.py:329
def __init__(self, filt=None, dataset=None, noDup=None)
Definition: functors.py:118
def _get_columnIndex(self, data)
Definition: functors.py:264
def _colsFromDict(self, colDict, columnIndex=None)
Definition: functors.py:184
def _get_data_columnLevels(self, data, columnIndex=None)
Definition: functors.py:138
def __call__(self, parq, dropna=False, **kwargs)
Definition: functors.py:885
def instFluxToNanojansky(self, instFlux, localCalib)
Definition: functors.py:1408
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
Definition: functors.py:1463
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
Definition: functors.py:1401
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
Definition: functors.py:1425
def instFluxToMagnitude(self, instFlux, localCalib)
Definition: functors.py:1446
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
Definition: functors.py:1086
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
Definition: functors.py:1093
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
Definition: functors.py:1122
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
Definition: functors.py:1148
def __init__(self, col1, col2, **kwargs)
Definition: functors.py:783
def __init__(self, *args, **kwargs)
Definition: functors.py:744
def __init__(self, col, calib=None, **kwargs)
Definition: functors.py:706
def dn2mag(self, dn, fluxMag0)
Definition: functors.py:1325
def dn2flux(self, dn, fluxMag0)
Definition: functors.py:1322
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
Definition: functors.py:1331
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
Definition: functors.py:1336
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
Definition: functors.py:1291
def __call__(self, catalog, **kwargs)
Definition: functors.py:651
def __init__(self, **kwargs)
Definition: functors.py:648
def __init__(self, colXX, colXY, colYY, **kwargs)
Definition: functors.py:1062
def mag_aware_eval(df, expr)
Definition: functors.py:521
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)
Definition: functors.py:15