24 from itertools
import product
29 import astropy.units
as u
32 from lsst.daf.butler
import DeferredDatasetHandle
33 from .parquetTable
import ParquetTable, MultilevelParquetTable
37 typeKey='functor', name=None):
38 """Initialize an object defined in a dictionary
40 The object needs to be importable as
41 f'{basePath}.{initDict[typeKey]}'
42 The positional and keyword arguments (if any) are contained in
43 "args" and "kwargs" entries in the dictionary, respectively.
44 This is used in `functors.CompositeFunctor.from_yaml` to initialize
45 a composite functor from a specification in a YAML file.
50 Dictionary describing object's initialization. Must contain
51 an entry keyed by ``typeKey`` that is the name of the object,
52 relative to ``basePath``.
54 Path relative to module in which ``initDict[typeKey]`` is defined.
56 Key of ``initDict`` that is the name of the object
57 (relative to `basePath`).
59 initDict = initDict.copy()
61 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
63 if 'args' in initDict:
64 args = initDict.pop(
'args')
65 if isinstance(args, str):
68 element = pythonType(*args, **initDict)
69 except Exception
as e:
70 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
71 raise type(e)(message, e.args)
76 """Define and execute a calculation on a ParquetTable
78 The `__call__` method accepts either a `ParquetTable` object or a
79 `DeferredDatasetHandle`, and returns the
80 result of the calculation as a single column. Each functor defines what
81 columns are needed for the calculation, and only these columns are read
82 from the `ParquetTable`.
84 The action of `__call__` consists of two steps: first, loading the
85 necessary columns from disk into memory as a `pandas.DataFrame` object;
86 and second, performing the computation on this dataframe and returning the
90 To define a new `Functor`, a subclass must define a `_func` method,
91 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
92 In addition, it must define the following attributes
94 * `_columns`: The columns necessary to perform the calculation
95 * `name`: A name appropriate for a figure axis label
96 * `shortname`: A name appropriate for use as a dictionary key
98 On initialization, a `Functor` should declare what band (`filt` kwarg)
99 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
100 applied to. This enables the `_get_data` method to extract the proper
101 columns from the parquet file. If not specified, the dataset will fall back
102 on the `_defaultDataset`attribute. If band is not specified and `dataset`
103 is anything other than `'ref'`, then an error will be raised when trying to
104 perform the calculation.
106 As currently implemented, `Functor` is only set up to expect a
107 dataset of the format of the `deepCoadd_obj` dataset; that is, a
108 dataframe with a multi-level column index,
109 with the levels of the column index being `band`,
110 `dataset`, and `column`. This is defined in the `_columnLevels` attribute,
111 as well as being implicit in the role of the `filt` and `dataset` attributes
112 defined at initialization. In addition, the `_get_data` method that reads
113 the dataframe from the `ParquetTable` will return a dataframe with column
114 index levels defined by the `_dfLevels` attribute; by default, this is
117 The `_columnLevels` and `_dfLevels` attributes should generally not need to
118 be changed, unless `_func` needs columns from multiple filters or datasets
119 to do the calculation.
120 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
121 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
122 it gets to have those levels in the column index.
127 Filter upon which to do the calculation
130 Dataset upon which to do the calculation
131 (e.g., 'ref', 'meas', 'forced_src').
135 _defaultDataset =
'ref'
136 _columnLevels = (
'band',
'dataset',
'column')
137 _dfLevels = (
'column',)
138 _defaultNoDup =
False
140 def __init__(self, filt=None, dataset=None, noDup=None):
147 if self.
_noDup_noDup
is not None:
154 """Columns required to perform calculation
156 if not hasattr(self,
'_columns'):
157 raise NotImplementedError(
'Must define columns property or _columns attribute')
160 def _get_data_columnLevels(self, data, columnIndex=None):
161 """Gets the names of the column index levels
163 This should only be called in the context of a multilevel table.
164 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
165 and with the gen3 `DeferredDatasetHandle`.
169 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
171 columnnIndex (optional): pandas `Index` object
172 if not passed, then it is read from the `DeferredDatasetHandle`
174 if isinstance(data, DeferredDatasetHandle):
175 if columnIndex
is None:
176 columnIndex = data.get(component=
"columns")
177 if columnIndex
is not None:
178 return columnIndex.names
179 if isinstance(data, MultilevelParquetTable):
180 return data.columnLevels
182 raise TypeError(f
"Unknown type for data: {type(data)}!")
184 def _get_data_columnLevelNames(self, data, columnIndex=None):
185 """Gets the content of each of the column levels for a multilevel table
187 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
189 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
191 if isinstance(data, DeferredDatasetHandle):
192 if columnIndex
is None:
193 columnIndex = data.get(component=
"columns")
194 if columnIndex
is not None:
195 columnLevels = columnIndex.names
197 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
198 for i, level
in enumerate(columnLevels)
200 return columnLevelNames
201 if isinstance(data, MultilevelParquetTable):
202 return data.columnLevelNames
204 raise TypeError(f
"Unknown type for data: {type(data)}!")
206 def _colsFromDict(self, colDict, columnIndex=None):
207 """Converts dictionary column specficiation to a list of columns
209 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
214 for i, lev
in enumerate(columnLevels):
216 if isinstance(colDict[lev], str):
217 new_colDict[lev] = [colDict[lev]]
219 new_colDict[lev] = colDict[lev]
221 new_colDict[lev] = columnIndex.levels[i]
223 levelCols = [new_colDict[lev]
for lev
in columnLevels]
224 cols = product(*levelCols)
228 """Returns columns needed by functor from multilevel dataset
230 To access tables with multilevel column structure, the `MultilevelParquetTable`
231 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
236 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
238 columnIndex (optional): pandas `Index` object
239 either passed or read in from `DeferredDatasetHandle`.
242 If true, then return a list of tuples rather than the column dictionary
243 specification. This is set to `True` by `CompositeFunctor` in order to be able to
244 combine columns from the various component functors.
247 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
248 columnIndex = data.get(component=
"columns")
253 if not set(columnLevels) == set(self.
_columnLevels_columnLevels):
255 "ParquetTable does not have the expected column levels. "
256 f
"Got {columnLevels}; expected {self._columnLevels}."
259 columnDict = {
'column': self.
columnscolumns,
260 'dataset': self.
datasetdataset}
261 if self.
filtfilt
is None:
263 if "band" in columnLevels:
264 if self.
datasetdataset ==
"ref":
265 columnDict[
"band"] = columnLevelNames[
"band"][0]
267 raise ValueError(f
"'filt' not set for functor {self.name}"
268 f
"(dataset {self.dataset}) "
270 "contains multiple filters in column index. "
271 "Set 'filt' or set 'dataset' to 'ref'.")
273 columnDict[
'band'] = self.
filtfilt
275 if isinstance(data, MultilevelParquetTable):
276 return data._colsFromDict(columnDict)
277 elif isinstance(data, DeferredDatasetHandle):
279 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
283 def _func(self, df, dropna=True):
284 raise NotImplementedError(
'Must define calculation on dataframe')
286 def _get_columnIndex(self, data):
287 """Return columnIndex
290 if isinstance(data, DeferredDatasetHandle):
291 return data.get(component=
"columns")
295 def _get_data(self, data):
296 """Retrieve dataframe necessary for calculation.
298 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
300 Returns dataframe upon which `self._func` can act.
302 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
304 if isinstance(data, pd.DataFrame):
309 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
312 if isinstance(data, ParquetTable)
and not is_multiLevel:
314 df = data.toDataFrame(columns=columns)
323 if isinstance(data, MultilevelParquetTable):
325 df = data.toDataFrame(columns=columns, droplevels=
False)
326 elif isinstance(data, DeferredDatasetHandle):
328 df = data.get(parameters={
"columns": columns})
336 def _setLevels(self, df):
337 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
338 df.columns = df.columns.droplevel(levelsToDrop)
341 def _dropna(self, vals):
347 vals = self.
_func_func(df)
349 vals = self.
failfail(df)
351 vals = self.
_dropna_dropna(vals)
356 """Computes difference between functor called on two different ParquetTable objects
358 return self(data1, **kwargs) - self(data2, **kwargs)
361 return pd.Series(np.full(len(df), np.nan), index=df.index)
365 """Full name of functor (suitable for figure labels)
367 return NotImplementedError
371 """Short name of functor (suitable for column name/dict key)
377 """Perform multiple calculations at once on a catalog
379 The role of a `CompositeFunctor` is to group together computations from
380 multiple functors. Instead of returning `pandas.Series` a
381 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
382 being the keys of `funcDict`.
384 The `columns` attribute of a `CompositeFunctor` is the union of all columns
385 in all the component functors.
387 A `CompositeFunctor` does not use a `_func` method itself; rather,
388 when a `CompositeFunctor` is called, all its columns are loaded
389 at once, and the resulting dataframe is passed to the `_func` method of each component
390 functor. This has the advantage of only doing I/O (reading from parquet file) once,
391 and works because each individual `_func` method of each component functor does not
392 care if there are *extra* columns in the dataframe being passed; only that it must contain
393 *at least* the `columns` it expects.
395 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
396 file specifying a collection of functors.
400 funcs : `dict` or `list`
401 Dictionary or list of functors. If a list, then it will be converted
402 into a dictonary according to the `.shortname` attribute of each functor.
409 if type(funcs) == dict:
412 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
414 self.
_filt_filt =
None
420 return self.
_filt_filt
425 for _, f
in self.
funcDictfuncDict.items():
427 self.
_filt_filt = filt
430 if isinstance(new, dict):
432 elif isinstance(new, CompositeFunctor):
435 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
443 return list(set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
452 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
460 """Apply the functor to the data table
464 data : `lsst.daf.butler.DeferredDatasetHandle`,
465 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
466 `lsst.pipe.tasks.parquetTable.ParquetTable`,
467 or `pandas.DataFrame`.
468 The table or a pointer to a table on disk from which columns can
474 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
480 if isinstance(data, MultilevelParquetTable):
482 df = data.toDataFrame(columns=columns, droplevels=
False)
483 elif isinstance(data, DeferredDatasetHandle):
485 df = data.get(parameters={
"columns": columns})
488 for k, f
in self.
funcDictfuncDict.items():
490 subdf = f._setLevels(
491 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
493 valDict[k] = f._func(subdf)
495 valDict[k] = f.fail(subdf)
498 if isinstance(data, DeferredDatasetHandle):
501 elif isinstance(data, pd.DataFrame):
508 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
511 valDf = pd.concat(valDict, axis=1)
513 print([(k, type(v))
for k, v
in valDict.items()])
516 if kwargs.get(
'dropna',
False):
517 valDf = valDf.dropna(how=
'any')
523 if renameRules
is None:
525 for old, new
in renameRules:
526 if col.startswith(old):
527 col = col.replace(old, new)
533 filename = os.path.expandvars(filename)
534 with open(filename)
as f:
535 translationDefinition = yaml.safe_load(f)
537 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
542 for func, val
in translationDefinition[
'funcs'].items():
545 if 'flag_rename_rules' in translationDefinition:
546 renameRules = translationDefinition[
'flag_rename_rules']
550 if 'refFlags' in translationDefinition:
551 for flag
in translationDefinition[
'refFlags']:
552 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
554 if 'forcedFlags' in translationDefinition:
555 for flag
in translationDefinition[
'forcedFlags']:
556 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
558 if 'flags' in translationDefinition:
559 for flag
in translationDefinition[
'flags']:
560 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
562 return cls(funcs, **kwargs)
566 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
568 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
572 df : pandas.DataFrame
573 Dataframe on which to evaluate expression.
579 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
580 val = df.eval(expr_new, truediv=
True)
582 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
583 val = df.eval(expr_new, truediv=
True)
588 """Arbitrary computation on a catalog
590 Column names (and thus the columns to be loaded from catalog) are found
591 by finding all words and trying to ignore all "math-y" words.
596 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
598 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
610 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
612 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
615 if not re.search(
'_instFlux$', c):
616 cols.append(f
'{c}_instFlux')
621 return list(set([c
for c
in cols
if c
not in not_a_col]))
628 """Get column with specified name
644 return df[self.
colcol]
648 """Return the value of the index for each object
651 columns = [
'coord_ra']
652 _defaultDataset =
'ref'
656 return pd.Series(df.index, index=df.index)
661 _allow_difference =
False
665 return pd.Series(df.index, index=df.index)
669 col =
'base_Footprint_nPix'
673 """Base class for coordinate column, in degrees
682 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
687 """Right Ascension, in degrees
693 super().
__init__(
'coord_ra', **kwargs)
696 return super().
__call__(catalog, **kwargs)
700 """Declination, in degrees
706 super().
__init__(
'coord_dec', **kwargs)
709 return super().
__call__(catalog, **kwargs)
713 if not col.endswith(
'_instFlux'):
719 if not col.endswith(
'_instFluxErr'):
720 col +=
'_instFluxErr'
725 """Compute calibrated magnitude
727 Takes a `calib` argument, which returns the flux at mag=0
728 as `calib.getFluxMag0()`. If not provided, then the default
729 `fluxMag0` is 63095734448.0194, which is default for HSC.
730 This default should be removed in DM-21955
732 This calculation hides warnings about invalid values and dividing by zero.
734 As for all functors, a `dataset` and `filt` kwarg should be provided upon
735 initialization. Unlike the default `Functor`, however, the default dataset
736 for a `Mag` is `'meas'`, rather than `'ref'`.
741 Name of flux column from which to compute magnitude. Can be parseable
742 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
743 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
745 calib : `lsst.afw.image.calib.Calib` (optional)
746 Object that knows zero point.
748 _defaultDataset =
'meas'
753 if calib
is not None:
757 self.
fluxMag0fluxMag0 = 63095734448.0194
766 with np.warnings.catch_warnings():
767 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
768 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
769 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
773 return f
'mag_{self.col}'
777 """Compute calibrated magnitude uncertainty
779 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
784 calib : `lsst.afw.image.calib.Calib` (optional)
785 Object that knows zero point.
790 if self.
calibcalib
is not None:
797 return [self.
colcol, self.
colcol +
'Err']
800 with np.warnings.catch_warnings():
801 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
802 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
804 x = df[fluxErrCol] / df[fluxCol]
806 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
811 return super().name +
'_err'
819 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
823 _defaultDataset =
'meas'
825 """Functor to calculate magnitude difference"""
834 return [self.
col1col1, self.
col2col2]
837 with np.warnings.catch_warnings():
838 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
839 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
840 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
844 return f
'(mag_{self.col1} - mag_{self.col2})'
848 return f
'magDiff_{self.col1}_{self.col2}'
852 """Compute the color between two filters
854 Computes color by initializing two different `Mag`
855 functors based on the `col` and filters provided, and
856 then returning the difference.
858 This is enabled by the `_func` expecting a dataframe with a
859 multilevel column index, with both `'band'` and `'column'`,
860 instead of just `'column'`, which is the `Functor` default.
861 This is controlled by the `_dfLevels` attribute.
863 Also of note, the default dataset for `Color` is `forced_src'`,
864 whereas for `Mag` it is `'meas'`.
869 Name of flux column from which to compute; same as would be passed to
870 `lsst.pipe.tasks.functors.Mag`.
873 Filters from which to compute magnitude difference.
874 Color computed is `Mag(filt2) - Mag(filt1)`.
876 _defaultDataset =
'forced_src'
877 _dfLevels = (
'band',
'column')
883 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
887 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
888 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
901 mag2 = self.mag2._func(df[self.filt2])
902 mag1 = self.mag1._func(df[self.filt1])
907 return [self.
mag1mag1.col, self.
mag2mag2.col]
914 return f
'{self.filt2} - {self.filt1} ({self.col})'
918 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
922 """Main function of this subclass is to override the dropna=True
925 _allow_difference =
False
930 return super().
__call__(parq, dropna=
False, **kwargs)
934 _columns = [
"base_ClassificationExtendedness_value"]
935 _column =
"base_ClassificationExtendedness_value"
940 test = (x < 0.5).astype(int)
941 test = test.mask(mask, 2)
945 categories = [
'galaxy',
'star', self.
_null_label_null_label]
946 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
947 index=x.index, name=
'label')
949 label = label.astype(str)
954 _columns = [
'numStarFlags']
955 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
961 n = len(x.unique()) - 1
963 labels = [
'noStar',
'maybe',
'star']
964 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
965 index=x.index, name=
'label')
968 label = label.astype(str)
974 name =
'Deconvolved Moments'
975 shortname =
'deconvolvedMoments'
976 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
977 "ext_shapeHSM_HsmSourceMoments_yy",
978 "base_SdssShape_xx",
"base_SdssShape_yy",
979 "ext_shapeHSM_HsmPsfMoments_xx",
980 "ext_shapeHSM_HsmPsfMoments_yy")
983 """Calculate deconvolved moments"""
984 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
985 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
987 hsm = np.ones(len(df))*np.nan
988 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
989 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
990 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
995 raise RuntimeError(
'No psf shape parameter found in catalog')
997 return hsm.where(np.isfinite(hsm), sdss) - psf
1001 """Functor to calculate SDSS trace radius size for sources"""
1002 name =
"SDSS Trace Size"
1003 shortname =
'sdssTrace'
1004 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1006 def _func(self, df):
1007 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1012 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1013 name =
"PSF - SDSS Trace Size"
1014 shortname =
'psf_sdssTrace'
1015 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1016 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1018 def _func(self, df):
1019 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1020 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1021 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1026 """Functor to calculate HSM trace radius size for sources"""
1027 name =
'HSM Trace Size'
1028 shortname =
'hsmTrace'
1029 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1030 "ext_shapeHSM_HsmSourceMoments_yy")
1032 def _func(self, df):
1033 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1034 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1039 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1040 name =
'PSF - HSM Trace Size'
1041 shortname =
'psf_HsmTrace'
1042 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1043 "ext_shapeHSM_HsmSourceMoments_yy",
1044 "ext_shapeHSM_HsmPsfMoments_xx",
1045 "ext_shapeHSM_HsmPsfMoments_yy")
1047 def _func(self, df):
1048 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1049 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1050 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1051 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1052 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1057 name =
'HSM Psf FWHM'
1058 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1061 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1063 def _func(self, df):
1065 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1069 name =
"Distortion Ellipticity (e1)"
1070 shortname =
"Distortion"
1083 def _func(self, df):
1088 name =
"Ellipticity e2"
1100 def _func(self, df):
1101 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1116 def _func(self, df):
1117 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1121 """Computations using the stored localWcs.
1123 name =
"LocalWcsOperations"
1138 """Compute the distance on the sphere from x2, y1 to x1, y1.
1146 cd11 : `pandas.Series`
1147 [1, 1] element of the local Wcs affine transform.
1148 cd11 : `pandas.Series`
1149 [1, 1] element of the local Wcs affine transform.
1150 cd12 : `pandas.Series`
1151 [1, 2] element of the local Wcs affine transform.
1152 cd21 : `pandas.Series`
1153 [2, 1] element of the local Wcs affine transform.
1154 cd22 : `pandas.Series`
1155 [2, 2] element of the local Wcs affine transform.
1160 RA and dec conversion of x and y given the local Wcs. Returned
1161 units are in radians.
1164 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1167 """Compute the local pixel scale conversion.
1171 ra1 : `pandas.Series`
1172 Ra of the first coordinate in radians.
1173 dec1 : `pandas.Series`
1174 Dec of the first coordinate in radians.
1175 ra2 : `pandas.Series`
1176 Ra of the second coordinate in radians.
1177 dec2 : `pandas.Series`
1178 Dec of the second coordinate in radians.
1182 dist : `pandas.Series`
1183 Distance on the sphere in radians.
1185 deltaDec = dec2 - dec1
1187 return 2 * np.arcsin(
1189 np.sin(deltaDec / 2) ** 2
1190 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1193 """Compute the distance on the sphere from x2, y1 to x1, y1.
1197 x1 : `pandas.Series`
1199 y1 : `pandas.Series`
1201 x2 : `pandas.Series`
1203 y2 : `pandas.Series`
1205 cd11 : `pandas.Series`
1206 [1, 1] element of the local Wcs affine transform.
1207 cd11 : `pandas.Series`
1208 [1, 1] element of the local Wcs affine transform.
1209 cd12 : `pandas.Series`
1210 [1, 2] element of the local Wcs affine transform.
1211 cd21 : `pandas.Series`
1212 [2, 1] element of the local Wcs affine transform.
1213 cd22 : `pandas.Series`
1214 [2, 2] element of the local Wcs affine transform.
1218 Distance : `pandas.Series`
1219 Arcseconds per pixel at the location of the local WC
1221 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1222 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1228 """Compute the local pixel scale from the stored CDMatrix.
1240 """Compute the local pixel to scale conversion in arcseconds.
1244 cd11 : `pandas.Series`
1245 [1, 1] element of the local Wcs affine transform in radians.
1246 cd11 : `pandas.Series`
1247 [1, 1] element of the local Wcs affine transform in radians.
1248 cd12 : `pandas.Series`
1249 [1, 2] element of the local Wcs affine transform in radians.
1250 cd21 : `pandas.Series`
1251 [2, 1] element of the local Wcs affine transform in radians.
1252 cd22 : `pandas.Series`
1253 [2, 2] element of the local Wcs affine transform in radians.
1257 pixScale : `pandas.Series`
1258 Arcseconds per pixel at the location of the local WC
1260 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1262 def _func(self, df):
1270 """Convert a value in units pixels squared to units arcseconds squared.
1289 return f
"{self.col}_asArcseconds"
1293 return [self.
colcol,
1299 def _func(self, df):
1307 """Convert a value in units pixels to units arcseconds.
1326 return f
"{self.col}_asArcsecondsSq"
1330 return [self.
colcol,
1336 def _func(self, df):
1341 return df[self.
colcol] * pixScale * pixScale
1345 name =
'Reference Band'
1346 shortname =
'refBand'
1350 return [
"merge_measurement_i",
1351 "merge_measurement_r",
1352 "merge_measurement_z",
1353 "merge_measurement_y",
1354 "merge_measurement_g"]
1356 def _func(self, df):
1357 def getFilterAliasName(row):
1359 colName = row.idxmax()
1360 return colName.replace(
'merge_measurement_',
'')
1362 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1367 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1368 LOG_AB_FLUX_SCALE = 12.56
1369 FIVE_OVER_2LOG10 = 1.085736204758129569
1373 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1379 if calib
is not None:
1389 return [self.
colcol]
1393 return f
'mag_{self.col}'
1397 if np.abs(a) < np.abs(b):
1402 return np.abs(a) * np.sqrt(1. + q*q)
1408 with np.warnings.catch_warnings():
1409 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1410 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1411 return -2.5 * np.log10(dn/fluxMag0)
1414 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1415 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1419 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1424 def _func(self, df):
1433 def _func(self, df):
1435 return pd.Series(retArr, index=df.index)
1439 def _func(self, df):
1448 def _func(self, df):
1450 return pd.Series(retArr, index=df.index)
1454 """Base class for calibrating the specified instrument flux column using
1455 the local photometric calibration.
1460 Name of the instrument flux column.
1461 instFluxErrCol : `str`
1462 Name of the assocated error columns for ``instFluxCol``.
1463 photoCalibCol : `str`
1464 Name of local calibration column.
1465 photoCalibErrCol : `str`
1466 Error associated with ``photoCalibCol``
1476 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1491 """Convert instrument flux to nanojanskys.
1495 instFlux : `numpy.ndarray` or `pandas.Series`
1496 Array of instrument flux measurements
1497 localCalib : `numpy.ndarray` or `pandas.Series`
1498 Array of local photometric calibration estimates.
1502 calibFlux : `numpy.ndarray` or `pandas.Series`
1503 Array of calibrated flux measurements.
1505 return instFlux * localCalib
1508 """Convert instrument flux to nanojanskys.
1512 instFlux : `numpy.ndarray` or `pandas.Series`
1513 Array of instrument flux measurements
1514 instFluxErr : `numpy.ndarray` or `pandas.Series`
1515 Errors on associated ``instFlux`` values
1516 localCalib : `numpy.ndarray` or `pandas.Series`
1517 Array of local photometric calibration estimates.
1518 localCalibErr : `numpy.ndarray` or `pandas.Series`
1519 Errors on associated ``localCalib`` values
1523 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1524 Errors on calibrated flux measurements.
1526 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1529 """Convert instrument flux to nanojanskys.
1533 instFlux : `numpy.ndarray` or `pandas.Series`
1534 Array of instrument flux measurements
1535 localCalib : `numpy.ndarray` or `pandas.Series`
1536 Array of local photometric calibration estimates.
1540 calibMag : `numpy.ndarray` or `pandas.Series`
1541 Array of calibrated AB magnitudes.
1546 """Convert instrument flux err to nanojanskys.
1550 instFlux : `numpy.ndarray` or `pandas.Series`
1551 Array of instrument flux measurements
1552 instFluxErr : `numpy.ndarray` or `pandas.Series`
1553 Errors on associated ``instFlux`` values
1554 localCalib : `numpy.ndarray` or `pandas.Series`
1555 Array of local photometric calibration estimates.
1556 localCalibErr : `numpy.ndarray` or `pandas.Series`
1557 Errors on associated ``localCalib`` values
1561 calibMagErr: `numpy.ndarray` or `pandas.Series`
1562 Error on calibrated AB magnitudes.
1565 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1569 """Compute calibrated fluxes using the local calibration value.
1585 return f
'flux_{self.instFluxCol}'
1587 def _func(self, df):
1592 """Compute calibrated flux errors using the local calibration value.
1609 return f
'fluxErr_{self.instFluxCol}'
1611 def _func(self, df):
1617 """Compute calibrated AB magnitudes using the local calibration value.
1633 return f
'mag_{self.instFluxCol}'
1635 def _func(self, df):
1641 """Compute calibrated AB magnitude errors using the local calibration value.
1658 return f
'magErr_{self.instFluxCol}'
1660 def _func(self, df):
1668 """Compute absolute mean of dipole fluxes.
1677 LocalDipoleMeanFluxErr
1679 LocalDipoleDiffFluxErr
1709 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1711 def _func(self, df):
1717 """Compute the error on the absolute mean of dipole fluxes.
1726 LocalDipoleMeanFluxErr
1728 LocalDipoleDiffFluxErr
1742 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1744 def _func(self, df):
1753 """Compute the absolute difference of dipole fluxes.
1755 Value is (abs(pos) - abs(neg))
1764 LocalDipoleMeanFluxErr
1766 LocalDipoleDiffFluxErr
1777 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1779 def _func(self, df):
1785 """Compute the error on the absolute difference of dipole fluxes.
1794 LocalDipoleMeanFluxErr
1796 LocalDipoleDiffFluxErr
1810 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1812 def _func(self, df):
1821 """Base class for returning the ratio of 2 columns.
1823 Can be used to compute a Signal to Noise ratio for any input flux.
1828 Name of the column to use at the numerator in the ratio
1830 Name of the column to use as the denominator in the ratio.
1846 return f
'ratio_{self.numerator}_{self.denominator}'
1848 def _func(self, df):
1849 with np.warnings.catch_warnings():
1850 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1851 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)