24 from itertools
import product
29 import astropy.units
as u
32 from lsst.daf.butler
import DeferredDatasetHandle
36 from .parquetTable
import ParquetTable, MultilevelParquetTable
40 typeKey='functor', name=None):
41 """Initialize an object defined in a dictionary
43 The object needs to be importable as
44 f'{basePath}.{initDict[typeKey]}'
45 The positional and keyword arguments (if any) are contained in
46 "args" and "kwargs" entries in the dictionary, respectively.
47 This is used in `functors.CompositeFunctor.from_yaml` to initialize
48 a composite functor from a specification in a YAML file.
53 Dictionary describing object's initialization. Must contain
54 an entry keyed by ``typeKey`` that is the name of the object,
55 relative to ``basePath``.
57 Path relative to module in which ``initDict[typeKey]`` is defined.
59 Key of ``initDict`` that is the name of the object
60 (relative to `basePath`).
62 initDict = initDict.copy()
64 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
66 if 'args' in initDict:
67 args = initDict.pop(
'args')
68 if isinstance(args, str):
71 element = pythonType(*args, **initDict)
72 except Exception
as e:
73 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
74 raise type(e)(message, e.args)
79 """Define and execute a calculation on a ParquetTable
81 The `__call__` method accepts either a `ParquetTable` object or a
82 `DeferredDatasetHandle`, and returns the
83 result of the calculation as a single column. Each functor defines what
84 columns are needed for the calculation, and only these columns are read
85 from the `ParquetTable`.
87 The action of `__call__` consists of two steps: first, loading the
88 necessary columns from disk into memory as a `pandas.DataFrame` object;
89 and second, performing the computation on this dataframe and returning the
93 To define a new `Functor`, a subclass must define a `_func` method,
94 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
95 In addition, it must define the following attributes
97 * `_columns`: The columns necessary to perform the calculation
98 * `name`: A name appropriate for a figure axis label
99 * `shortname`: A name appropriate for use as a dictionary key
101 On initialization, a `Functor` should declare what band (`filt` kwarg)
102 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
103 applied to. This enables the `_get_data` method to extract the proper
104 columns from the parquet file. If not specified, the dataset will fall back
105 on the `_defaultDataset`attribute. If band is not specified and `dataset`
106 is anything other than `'ref'`, then an error will be raised when trying to
107 perform the calculation.
109 Originally, `Functor` was set up to expect
110 datasets formatted like the `deepCoadd_obj` dataset; that is, a
111 dataframe with a multi-level column index, with the levels of the
112 column index being `band`, `dataset`, and `column`.
113 It has since been generalized to apply to dataframes without mutli-level
114 indices and multi-level indices with just `dataset` and `column` levels.
115 In addition, the `_get_data` method that reads
116 the dataframe from the `ParquetTable` will return a dataframe with column
117 index levels defined by the `_dfLevels` attribute; by default, this is
120 The `_dfLevels` attributes should generally not need to
121 be changed, unless `_func` needs columns from multiple filters or datasets
122 to do the calculation.
123 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
124 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
125 it gets to have those levels in the column index.
130 Filter upon which to do the calculation
133 Dataset upon which to do the calculation
134 (e.g., 'ref', 'meas', 'forced_src').
138 _defaultDataset =
'ref'
139 _dfLevels = (
'column',)
140 _defaultNoDup =
False
142 def __init__(self, filt=None, dataset=None, noDup=None):
149 if self.
_noDup_noDup
is not None:
156 """Columns required to perform calculation
158 if not hasattr(self,
'_columns'):
159 raise NotImplementedError(
'Must define columns property or _columns attribute')
162 def _get_data_columnLevels(self, data, columnIndex=None):
163 """Gets the names of the column index levels
165 This should only be called in the context of a multilevel table.
166 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
167 and with the gen3 `DeferredDatasetHandle`.
171 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
173 columnnIndex (optional): pandas `Index` object
174 if not passed, then it is read from the `DeferredDatasetHandle`
176 if isinstance(data, DeferredDatasetHandle):
177 if columnIndex
is None:
178 columnIndex = data.get(component=
"columns")
179 if columnIndex
is not None:
180 return columnIndex.names
181 if isinstance(data, MultilevelParquetTable):
182 return data.columnLevels
184 raise TypeError(f
"Unknown type for data: {type(data)}!")
186 def _get_data_columnLevelNames(self, data, columnIndex=None):
187 """Gets the content of each of the column levels for a multilevel table
189 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
191 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
193 if isinstance(data, DeferredDatasetHandle):
194 if columnIndex
is None:
195 columnIndex = data.get(component=
"columns")
196 if columnIndex
is not None:
197 columnLevels = columnIndex.names
199 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
200 for i, level
in enumerate(columnLevels)
202 return columnLevelNames
203 if isinstance(data, MultilevelParquetTable):
204 return data.columnLevelNames
206 raise TypeError(f
"Unknown type for data: {type(data)}!")
208 def _colsFromDict(self, colDict, columnIndex=None):
209 """Converts dictionary column specficiation to a list of columns
211 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
216 for i, lev
in enumerate(columnLevels):
218 if isinstance(colDict[lev], str):
219 new_colDict[lev] = [colDict[lev]]
221 new_colDict[lev] = colDict[lev]
223 new_colDict[lev] = columnIndex.levels[i]
225 levelCols = [new_colDict[lev]
for lev
in columnLevels]
226 cols = product(*levelCols)
230 """Returns columns needed by functor from multilevel dataset
232 To access tables with multilevel column structure, the `MultilevelParquetTable`
233 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
238 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
240 columnIndex (optional): pandas `Index` object
241 either passed or read in from `DeferredDatasetHandle`.
244 If true, then return a list of tuples rather than the column dictionary
245 specification. This is set to `True` by `CompositeFunctor` in order to be able to
246 combine columns from the various component functors.
249 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
250 columnIndex = data.get(component=
"columns")
255 columnDict = {
'column': self.
columnscolumns,
256 'dataset': self.
datasetdataset}
257 if self.
filtfilt
is None:
259 if "band" in columnLevels:
260 if self.
datasetdataset ==
"ref":
261 columnDict[
"band"] = columnLevelNames[
"band"][0]
263 raise ValueError(f
"'filt' not set for functor {self.name}"
264 f
"(dataset {self.dataset}) "
266 "contains multiple filters in column index. "
267 "Set 'filt' or set 'dataset' to 'ref'.")
269 columnDict[
'band'] = self.
filtfilt
271 if isinstance(data, MultilevelParquetTable):
272 return data._colsFromDict(columnDict)
273 elif isinstance(data, DeferredDatasetHandle):
275 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
279 def _func(self, df, dropna=True):
280 raise NotImplementedError(
'Must define calculation on dataframe')
282 def _get_columnIndex(self, data):
283 """Return columnIndex
286 if isinstance(data, DeferredDatasetHandle):
287 return data.get(component=
"columns")
291 def _get_data(self, data):
292 """Retrieve dataframe necessary for calculation.
294 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
296 Returns dataframe upon which `self._func` can act.
298 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
300 if isinstance(data, pd.DataFrame):
305 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
308 if isinstance(data, ParquetTable)
and not is_multiLevel:
310 df = data.toDataFrame(columns=columns)
319 if isinstance(data, MultilevelParquetTable):
321 df = data.toDataFrame(columns=columns, droplevels=
False)
322 elif isinstance(data, DeferredDatasetHandle):
324 df = data.get(parameters={
"columns": columns})
332 def _setLevels(self, df):
333 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
334 df.columns = df.columns.droplevel(levelsToDrop)
337 def _dropna(self, vals):
343 vals = self.
_func_func(df)
345 vals = self.
failfail(df)
347 vals = self.
_dropna_dropna(vals)
352 """Computes difference between functor called on two different ParquetTable objects
354 return self(data1, **kwargs) - self(data2, **kwargs)
357 return pd.Series(np.full(len(df), np.nan), index=df.index)
361 """Full name of functor (suitable for figure labels)
363 return NotImplementedError
367 """Short name of functor (suitable for column name/dict key)
373 """Perform multiple calculations at once on a catalog
375 The role of a `CompositeFunctor` is to group together computations from
376 multiple functors. Instead of returning `pandas.Series` a
377 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
378 being the keys of `funcDict`.
380 The `columns` attribute of a `CompositeFunctor` is the union of all columns
381 in all the component functors.
383 A `CompositeFunctor` does not use a `_func` method itself; rather,
384 when a `CompositeFunctor` is called, all its columns are loaded
385 at once, and the resulting dataframe is passed to the `_func` method of each component
386 functor. This has the advantage of only doing I/O (reading from parquet file) once,
387 and works because each individual `_func` method of each component functor does not
388 care if there are *extra* columns in the dataframe being passed; only that it must contain
389 *at least* the `columns` it expects.
391 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
392 file specifying a collection of functors.
396 funcs : `dict` or `list`
397 Dictionary or list of functors. If a list, then it will be converted
398 into a dictonary according to the `.shortname` attribute of each functor.
405 if type(funcs) == dict:
408 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
410 self.
_filt_filt =
None
416 return self.
_filt_filt
421 for _, f
in self.
funcDictfuncDict.items():
423 self.
_filt_filt = filt
426 if isinstance(new, dict):
428 elif isinstance(new, CompositeFunctor):
431 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
439 return list(set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
448 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
456 """Apply the functor to the data table
460 data : `lsst.daf.butler.DeferredDatasetHandle`,
461 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
462 `lsst.pipe.tasks.parquetTable.ParquetTable`,
463 or `pandas.DataFrame`.
464 The table or a pointer to a table on disk from which columns can
470 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
476 if isinstance(data, MultilevelParquetTable):
478 df = data.toDataFrame(columns=columns, droplevels=
False)
479 elif isinstance(data, DeferredDatasetHandle):
481 df = data.get(parameters={
"columns": columns})
484 for k, f
in self.
funcDictfuncDict.items():
486 subdf = f._setLevels(
487 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
489 valDict[k] = f._func(subdf)
490 except Exception
as e:
492 valDict[k] = f.fail(subdf)
497 if isinstance(data, DeferredDatasetHandle):
500 elif isinstance(data, pd.DataFrame):
507 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
510 for name, colVal
in valDict.items():
511 if len(colVal.shape) != 1:
512 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
513 "It is shaped %s and type %s." % (name, colVal.shape, type(colVal)))
516 valDf = pd.concat(valDict, axis=1)
518 print([(k, type(v))
for k, v
in valDict.items()])
521 if kwargs.get(
'dropna',
False):
522 valDf = valDf.dropna(how=
'any')
528 if renameRules
is None:
530 for old, new
in renameRules:
531 if col.startswith(old):
532 col = col.replace(old, new)
538 filename = os.path.expandvars(filename)
539 with open(filename)
as f:
540 translationDefinition = yaml.safe_load(f)
542 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
547 for func, val
in translationDefinition[
'funcs'].items():
550 if 'flag_rename_rules' in translationDefinition:
551 renameRules = translationDefinition[
'flag_rename_rules']
555 if 'calexpFlags' in translationDefinition:
556 for flag
in translationDefinition[
'calexpFlags']:
557 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
559 if 'refFlags' in translationDefinition:
560 for flag
in translationDefinition[
'refFlags']:
561 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
563 if 'forcedFlags' in translationDefinition:
564 for flag
in translationDefinition[
'forcedFlags']:
565 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
567 if 'flags' in translationDefinition:
568 for flag
in translationDefinition[
'flags']:
569 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
571 return cls(funcs, **kwargs)
575 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
577 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
581 df : pandas.DataFrame
582 Dataframe on which to evaluate expression.
588 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
589 val = df.eval(expr_new, truediv=
True)
591 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
592 val = df.eval(expr_new, truediv=
True)
597 """Arbitrary computation on a catalog
599 Column names (and thus the columns to be loaded from catalog) are found
600 by finding all words and trying to ignore all "math-y" words.
605 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
607 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
619 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
621 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
624 if not re.search(
'_instFlux$', c):
625 cols.append(f
'{c}_instFlux')
630 return list(set([c
for c
in cols
if c
not in not_a_col]))
637 """Get column with specified name
653 return df[self.
colcol]
657 """Return the value of the index for each object
660 columns = [
'coord_ra']
661 _defaultDataset =
'ref'
665 return pd.Series(df.index, index=df.index)
670 _allow_difference =
False
674 return pd.Series(df.index, index=df.index)
678 col =
'base_Footprint_nPix'
682 """Base class for coordinate column, in degrees
691 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
696 """Right Ascension, in degrees
702 super().
__init__(
'coord_ra', **kwargs)
705 return super().
__call__(catalog, **kwargs)
709 """Declination, in degrees
715 super().
__init__(
'coord_dec', **kwargs)
718 return super().
__call__(catalog, **kwargs)
722 """Compute the level 20 HtmIndex for the catalog.
737 def computePixel(row):
746 return self.
pixelatorpixelator.index(sphPoint.getVector())
748 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
752 if not col.endswith(
'_instFlux'):
758 if not col.endswith(
'_instFluxErr'):
759 col +=
'_instFluxErr'
764 """Compute calibrated magnitude
766 Takes a `calib` argument, which returns the flux at mag=0
767 as `calib.getFluxMag0()`. If not provided, then the default
768 `fluxMag0` is 63095734448.0194, which is default for HSC.
769 This default should be removed in DM-21955
771 This calculation hides warnings about invalid values and dividing by zero.
773 As for all functors, a `dataset` and `filt` kwarg should be provided upon
774 initialization. Unlike the default `Functor`, however, the default dataset
775 for a `Mag` is `'meas'`, rather than `'ref'`.
780 Name of flux column from which to compute magnitude. Can be parseable
781 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
782 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
784 calib : `lsst.afw.image.calib.Calib` (optional)
785 Object that knows zero point.
787 _defaultDataset =
'meas'
792 if calib
is not None:
796 self.
fluxMag0fluxMag0 = 63095734448.0194
805 with np.warnings.catch_warnings():
806 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
807 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
808 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
812 return f
'mag_{self.col}'
816 """Compute calibrated magnitude uncertainty
818 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
823 calib : `lsst.afw.image.calib.Calib` (optional)
824 Object that knows zero point.
829 if self.
calibcalib
is not None:
836 return [self.
colcol, self.
colcol +
'Err']
839 with np.warnings.catch_warnings():
840 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
841 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
843 x = df[fluxErrCol] / df[fluxCol]
845 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
850 return super().name +
'_err'
858 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
862 _defaultDataset =
'meas'
864 """Functor to calculate magnitude difference"""
873 return [self.
col1col1, self.
col2col2]
876 with np.warnings.catch_warnings():
877 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
878 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
879 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
883 return f
'(mag_{self.col1} - mag_{self.col2})'
887 return f
'magDiff_{self.col1}_{self.col2}'
891 """Compute the color between two filters
893 Computes color by initializing two different `Mag`
894 functors based on the `col` and filters provided, and
895 then returning the difference.
897 This is enabled by the `_func` expecting a dataframe with a
898 multilevel column index, with both `'band'` and `'column'`,
899 instead of just `'column'`, which is the `Functor` default.
900 This is controlled by the `_dfLevels` attribute.
902 Also of note, the default dataset for `Color` is `forced_src'`,
903 whereas for `Mag` it is `'meas'`.
908 Name of flux column from which to compute; same as would be passed to
909 `lsst.pipe.tasks.functors.Mag`.
912 Filters from which to compute magnitude difference.
913 Color computed is `Mag(filt2) - Mag(filt1)`.
915 _defaultDataset =
'forced_src'
916 _dfLevels = (
'band',
'column')
922 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
926 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
927 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
940 mag2 = self.mag2._func(df[self.filt2])
941 mag1 = self.mag1._func(df[self.filt1])
946 return [self.
mag1mag1.col, self.
mag2mag2.col]
953 return f
'{self.filt2} - {self.filt1} ({self.col})'
957 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
961 """Main function of this subclass is to override the dropna=True
964 _allow_difference =
False
969 return super().
__call__(parq, dropna=
False, **kwargs)
973 _columns = [
"base_ClassificationExtendedness_value"]
974 _column =
"base_ClassificationExtendedness_value"
979 test = (x < 0.5).astype(int)
980 test = test.mask(mask, 2)
984 categories = [
'galaxy',
'star', self.
_null_label_null_label]
985 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
986 index=x.index, name=
'label')
988 label = label.astype(str)
993 _columns = [
'numStarFlags']
994 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
1000 n = len(x.unique()) - 1
1002 labels = [
'noStar',
'maybe',
'star']
1003 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
1004 index=x.index, name=
'label')
1007 label = label.astype(str)
1013 name =
'Deconvolved Moments'
1014 shortname =
'deconvolvedMoments'
1015 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1016 "ext_shapeHSM_HsmSourceMoments_yy",
1017 "base_SdssShape_xx",
"base_SdssShape_yy",
1018 "ext_shapeHSM_HsmPsfMoments_xx",
1019 "ext_shapeHSM_HsmPsfMoments_yy")
1021 def _func(self, df):
1022 """Calculate deconvolved moments"""
1023 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1024 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1026 hsm = np.ones(len(df))*np.nan
1027 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1028 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1029 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1034 raise RuntimeError(
'No psf shape parameter found in catalog')
1036 return hsm.where(np.isfinite(hsm), sdss) - psf
1040 """Functor to calculate SDSS trace radius size for sources"""
1041 name =
"SDSS Trace Size"
1042 shortname =
'sdssTrace'
1043 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1045 def _func(self, df):
1046 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1051 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1052 name =
"PSF - SDSS Trace Size"
1053 shortname =
'psf_sdssTrace'
1054 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1055 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1057 def _func(self, df):
1058 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1059 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1060 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1065 """Functor to calculate HSM trace radius size for sources"""
1066 name =
'HSM Trace Size'
1067 shortname =
'hsmTrace'
1068 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1069 "ext_shapeHSM_HsmSourceMoments_yy")
1071 def _func(self, df):
1072 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1073 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1078 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1079 name =
'PSF - HSM Trace Size'
1080 shortname =
'psf_HsmTrace'
1081 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1082 "ext_shapeHSM_HsmSourceMoments_yy",
1083 "ext_shapeHSM_HsmPsfMoments_xx",
1084 "ext_shapeHSM_HsmPsfMoments_yy")
1086 def _func(self, df):
1087 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1088 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1089 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1090 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1091 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1096 name =
'HSM Psf FWHM'
1097 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1100 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1102 def _func(self, df):
1104 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1108 name =
"Distortion Ellipticity (e1)"
1109 shortname =
"Distortion"
1122 def _func(self, df):
1127 name =
"Ellipticity e2"
1139 def _func(self, df):
1140 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1155 def _func(self, df):
1156 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1160 """Computations using the stored localWcs.
1162 name =
"LocalWcsOperations"
1177 """Compute the distance on the sphere from x2, y1 to x1, y1.
1185 cd11 : `pandas.Series`
1186 [1, 1] element of the local Wcs affine transform.
1187 cd11 : `pandas.Series`
1188 [1, 1] element of the local Wcs affine transform.
1189 cd12 : `pandas.Series`
1190 [1, 2] element of the local Wcs affine transform.
1191 cd21 : `pandas.Series`
1192 [2, 1] element of the local Wcs affine transform.
1193 cd22 : `pandas.Series`
1194 [2, 2] element of the local Wcs affine transform.
1199 RA and dec conversion of x and y given the local Wcs. Returned
1200 units are in radians.
1203 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1206 """Compute the local pixel scale conversion.
1210 ra1 : `pandas.Series`
1211 Ra of the first coordinate in radians.
1212 dec1 : `pandas.Series`
1213 Dec of the first coordinate in radians.
1214 ra2 : `pandas.Series`
1215 Ra of the second coordinate in radians.
1216 dec2 : `pandas.Series`
1217 Dec of the second coordinate in radians.
1221 dist : `pandas.Series`
1222 Distance on the sphere in radians.
1224 deltaDec = dec2 - dec1
1226 return 2 * np.arcsin(
1228 np.sin(deltaDec / 2) ** 2
1229 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1232 """Compute the distance on the sphere from x2, y1 to x1, y1.
1236 x1 : `pandas.Series`
1238 y1 : `pandas.Series`
1240 x2 : `pandas.Series`
1242 y2 : `pandas.Series`
1244 cd11 : `pandas.Series`
1245 [1, 1] element of the local Wcs affine transform.
1246 cd11 : `pandas.Series`
1247 [1, 1] element of the local Wcs affine transform.
1248 cd12 : `pandas.Series`
1249 [1, 2] element of the local Wcs affine transform.
1250 cd21 : `pandas.Series`
1251 [2, 1] element of the local Wcs affine transform.
1252 cd22 : `pandas.Series`
1253 [2, 2] element of the local Wcs affine transform.
1257 Distance : `pandas.Series`
1258 Arcseconds per pixel at the location of the local WC
1260 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1261 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1267 """Compute the local pixel scale from the stored CDMatrix.
1279 """Compute the local pixel to scale conversion in arcseconds.
1283 cd11 : `pandas.Series`
1284 [1, 1] element of the local Wcs affine transform in radians.
1285 cd11 : `pandas.Series`
1286 [1, 1] element of the local Wcs affine transform in radians.
1287 cd12 : `pandas.Series`
1288 [1, 2] element of the local Wcs affine transform in radians.
1289 cd21 : `pandas.Series`
1290 [2, 1] element of the local Wcs affine transform in radians.
1291 cd22 : `pandas.Series`
1292 [2, 2] element of the local Wcs affine transform in radians.
1296 pixScale : `pandas.Series`
1297 Arcseconds per pixel at the location of the local WC
1299 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1301 def _func(self, df):
1309 """Convert a value in units pixels squared to units arcseconds squared.
1328 return f
"{self.col}_asArcseconds"
1332 return [self.
colcol,
1338 def _func(self, df):
1346 """Convert a value in units pixels to units arcseconds.
1365 return f
"{self.col}_asArcsecondsSq"
1369 return [self.
colcol,
1375 def _func(self, df):
1380 return df[self.
colcol] * pixScale * pixScale
1384 name =
'Reference Band'
1385 shortname =
'refBand'
1389 return [
"merge_measurement_i",
1390 "merge_measurement_r",
1391 "merge_measurement_z",
1392 "merge_measurement_y",
1393 "merge_measurement_g",
1394 "merge_measurement_u"]
1396 def _func(self, df: pd.DataFrame) -> pd.Series:
1397 def getFilterAliasName(row):
1399 colName = row.idxmax()
1400 return colName.replace(
'merge_measurement_',
'')
1403 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1,
1404 result_type=
'reduce').astype(
'object')
1409 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1410 LOG_AB_FLUX_SCALE = 12.56
1411 FIVE_OVER_2LOG10 = 1.085736204758129569
1415 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1421 if calib
is not None:
1431 return [self.
colcol]
1435 return f
'mag_{self.col}'
1439 if np.abs(a) < np.abs(b):
1444 return np.abs(a) * np.sqrt(1. + q*q)
1450 with np.warnings.catch_warnings():
1451 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1452 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1453 return -2.5 * np.log10(dn/fluxMag0)
1456 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1457 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1461 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1466 def _func(self, df):
1475 def _func(self, df):
1477 return pd.Series(retArr, index=df.index)
1481 def _func(self, df):
1490 def _func(self, df):
1492 return pd.Series(retArr, index=df.index)
1496 """Base class for calibrating the specified instrument flux column using
1497 the local photometric calibration.
1502 Name of the instrument flux column.
1503 instFluxErrCol : `str`
1504 Name of the assocated error columns for ``instFluxCol``.
1505 photoCalibCol : `str`
1506 Name of local calibration column.
1507 photoCalibErrCol : `str`
1508 Error associated with ``photoCalibCol``
1518 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1533 """Convert instrument flux to nanojanskys.
1537 instFlux : `numpy.ndarray` or `pandas.Series`
1538 Array of instrument flux measurements
1539 localCalib : `numpy.ndarray` or `pandas.Series`
1540 Array of local photometric calibration estimates.
1544 calibFlux : `numpy.ndarray` or `pandas.Series`
1545 Array of calibrated flux measurements.
1547 return instFlux * localCalib
1550 """Convert instrument flux to nanojanskys.
1554 instFlux : `numpy.ndarray` or `pandas.Series`
1555 Array of instrument flux measurements
1556 instFluxErr : `numpy.ndarray` or `pandas.Series`
1557 Errors on associated ``instFlux`` values
1558 localCalib : `numpy.ndarray` or `pandas.Series`
1559 Array of local photometric calibration estimates.
1560 localCalibErr : `numpy.ndarray` or `pandas.Series`
1561 Errors on associated ``localCalib`` values
1565 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1566 Errors on calibrated flux measurements.
1568 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1571 """Convert instrument flux to nanojanskys.
1575 instFlux : `numpy.ndarray` or `pandas.Series`
1576 Array of instrument flux measurements
1577 localCalib : `numpy.ndarray` or `pandas.Series`
1578 Array of local photometric calibration estimates.
1582 calibMag : `numpy.ndarray` or `pandas.Series`
1583 Array of calibrated AB magnitudes.
1588 """Convert instrument flux err to nanojanskys.
1592 instFlux : `numpy.ndarray` or `pandas.Series`
1593 Array of instrument flux measurements
1594 instFluxErr : `numpy.ndarray` or `pandas.Series`
1595 Errors on associated ``instFlux`` values
1596 localCalib : `numpy.ndarray` or `pandas.Series`
1597 Array of local photometric calibration estimates.
1598 localCalibErr : `numpy.ndarray` or `pandas.Series`
1599 Errors on associated ``localCalib`` values
1603 calibMagErr: `numpy.ndarray` or `pandas.Series`
1604 Error on calibrated AB magnitudes.
1607 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1611 """Compute calibrated fluxes using the local calibration value.
1627 return f
'flux_{self.instFluxCol}'
1629 def _func(self, df):
1634 """Compute calibrated flux errors using the local calibration value.
1651 return f
'fluxErr_{self.instFluxCol}'
1653 def _func(self, df):
1659 """Compute calibrated AB magnitudes using the local calibration value.
1675 return f
'mag_{self.instFluxCol}'
1677 def _func(self, df):
1683 """Compute calibrated AB magnitude errors using the local calibration value.
1700 return f
'magErr_{self.instFluxCol}'
1702 def _func(self, df):
1710 """Compute absolute mean of dipole fluxes.
1719 LocalDipoleMeanFluxErr
1721 LocalDipoleDiffFluxErr
1751 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1753 def _func(self, df):
1759 """Compute the error on the absolute mean of dipole fluxes.
1768 LocalDipoleMeanFluxErr
1770 LocalDipoleDiffFluxErr
1784 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1786 def _func(self, df):
1795 """Compute the absolute difference of dipole fluxes.
1797 Value is (abs(pos) - abs(neg))
1806 LocalDipoleMeanFluxErr
1808 LocalDipoleDiffFluxErr
1819 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1821 def _func(self, df):
1827 """Compute the error on the absolute difference of dipole fluxes.
1836 LocalDipoleMeanFluxErr
1838 LocalDipoleDiffFluxErr
1852 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1854 def _func(self, df):
1863 """Base class for returning the ratio of 2 columns.
1865 Can be used to compute a Signal to Noise ratio for any input flux.
1870 Name of the column to use at the numerator in the ratio
1872 Name of the column to use as the denominator in the ratio.
1888 return f
'ratio_{self.numerator}_{self.denominator}'
1890 def _func(self, df):
1891 with np.warnings.catch_warnings():
1892 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1893 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)