24 from itertools
import product
29 import astropy.units
as u
32 from lsst.daf.butler
import DeferredDatasetHandle
36 from .parquetTable
import ParquetTable, MultilevelParquetTable
40 typeKey='functor', name=None):
41 """Initialize an object defined in a dictionary
43 The object needs to be importable as
44 f'{basePath}.{initDict[typeKey]}'
45 The positional and keyword arguments (if any) are contained in
46 "args" and "kwargs" entries in the dictionary, respectively.
47 This is used in `functors.CompositeFunctor.from_yaml` to initialize
48 a composite functor from a specification in a YAML file.
53 Dictionary describing object's initialization. Must contain
54 an entry keyed by ``typeKey`` that is the name of the object,
55 relative to ``basePath``.
57 Path relative to module in which ``initDict[typeKey]`` is defined.
59 Key of ``initDict`` that is the name of the object
60 (relative to `basePath`).
62 initDict = initDict.copy()
64 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
66 if 'args' in initDict:
67 args = initDict.pop(
'args')
68 if isinstance(args, str):
71 element = pythonType(*args, **initDict)
72 except Exception
as e:
73 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
74 raise type(e)(message, e.args)
79 """Define and execute a calculation on a ParquetTable
81 The `__call__` method accepts either a `ParquetTable` object or a
82 `DeferredDatasetHandle`, and returns the
83 result of the calculation as a single column. Each functor defines what
84 columns are needed for the calculation, and only these columns are read
85 from the `ParquetTable`.
87 The action of `__call__` consists of two steps: first, loading the
88 necessary columns from disk into memory as a `pandas.DataFrame` object;
89 and second, performing the computation on this dataframe and returning the
93 To define a new `Functor`, a subclass must define a `_func` method,
94 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
95 In addition, it must define the following attributes
97 * `_columns`: The columns necessary to perform the calculation
98 * `name`: A name appropriate for a figure axis label
99 * `shortname`: A name appropriate for use as a dictionary key
101 On initialization, a `Functor` should declare what band (`filt` kwarg)
102 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
103 applied to. This enables the `_get_data` method to extract the proper
104 columns from the parquet file. If not specified, the dataset will fall back
105 on the `_defaultDataset`attribute. If band is not specified and `dataset`
106 is anything other than `'ref'`, then an error will be raised when trying to
107 perform the calculation.
109 Originally, `Functor` was set up to expect
110 datasets formatted like the `deepCoadd_obj` dataset; that is, a
111 dataframe with a multi-level column index, with the levels of the
112 column index being `band`, `dataset`, and `column`.
113 It has since been generalized to apply to dataframes without mutli-level
114 indices and multi-level indices with just `dataset` and `column` levels.
115 In addition, the `_get_data` method that reads
116 the dataframe from the `ParquetTable` will return a dataframe with column
117 index levels defined by the `_dfLevels` attribute; by default, this is
120 The `_dfLevels` attributes should generally not need to
121 be changed, unless `_func` needs columns from multiple filters or datasets
122 to do the calculation.
123 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
124 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
125 it gets to have those levels in the column index.
130 Filter upon which to do the calculation
133 Dataset upon which to do the calculation
134 (e.g., 'ref', 'meas', 'forced_src').
138 _defaultDataset =
'ref'
139 _dfLevels = (
'column',)
140 _defaultNoDup =
False
142 def __init__(self, filt=None, dataset=None, noDup=None):
149 if self.
_noDup_noDup
is not None:
156 """Columns required to perform calculation
158 if not hasattr(self,
'_columns'):
159 raise NotImplementedError(
'Must define columns property or _columns attribute')
162 def _get_data_columnLevels(self, data, columnIndex=None):
163 """Gets the names of the column index levels
165 This should only be called in the context of a multilevel table.
166 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
167 and with the gen3 `DeferredDatasetHandle`.
171 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
173 columnnIndex (optional): pandas `Index` object
174 if not passed, then it is read from the `DeferredDatasetHandle`
176 if isinstance(data, DeferredDatasetHandle):
177 if columnIndex
is None:
178 columnIndex = data.get(component=
"columns")
179 if columnIndex
is not None:
180 return columnIndex.names
181 if isinstance(data, MultilevelParquetTable):
182 return data.columnLevels
184 raise TypeError(f
"Unknown type for data: {type(data)}!")
186 def _get_data_columnLevelNames(self, data, columnIndex=None):
187 """Gets the content of each of the column levels for a multilevel table
189 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
191 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
193 if isinstance(data, DeferredDatasetHandle):
194 if columnIndex
is None:
195 columnIndex = data.get(component=
"columns")
196 if columnIndex
is not None:
197 columnLevels = columnIndex.names
199 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
200 for i, level
in enumerate(columnLevels)
202 return columnLevelNames
203 if isinstance(data, MultilevelParquetTable):
204 return data.columnLevelNames
206 raise TypeError(f
"Unknown type for data: {type(data)}!")
208 def _colsFromDict(self, colDict, columnIndex=None):
209 """Converts dictionary column specficiation to a list of columns
211 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
216 for i, lev
in enumerate(columnLevels):
218 if isinstance(colDict[lev], str):
219 new_colDict[lev] = [colDict[lev]]
221 new_colDict[lev] = colDict[lev]
223 new_colDict[lev] = columnIndex.levels[i]
225 levelCols = [new_colDict[lev]
for lev
in columnLevels]
226 cols = product(*levelCols)
230 """Returns columns needed by functor from multilevel dataset
232 To access tables with multilevel column structure, the `MultilevelParquetTable`
233 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
238 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
240 columnIndex (optional): pandas `Index` object
241 either passed or read in from `DeferredDatasetHandle`.
244 If true, then return a list of tuples rather than the column dictionary
245 specification. This is set to `True` by `CompositeFunctor` in order to be able to
246 combine columns from the various component functors.
249 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
250 columnIndex = data.get(component=
"columns")
255 columnDict = {
'column': self.
columnscolumns,
256 'dataset': self.
datasetdataset}
257 if self.
filtfilt
is None:
259 if "band" in columnLevels:
260 if self.
datasetdataset ==
"ref":
261 columnDict[
"band"] = columnLevelNames[
"band"][0]
263 raise ValueError(f
"'filt' not set for functor {self.name}"
264 f
"(dataset {self.dataset}) "
266 "contains multiple filters in column index. "
267 "Set 'filt' or set 'dataset' to 'ref'.")
269 columnDict[
'band'] = self.
filtfilt
271 if isinstance(data, MultilevelParquetTable):
272 return data._colsFromDict(columnDict)
273 elif isinstance(data, DeferredDatasetHandle):
275 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
279 def _func(self, df, dropna=True):
280 raise NotImplementedError(
'Must define calculation on dataframe')
282 def _get_columnIndex(self, data):
283 """Return columnIndex
286 if isinstance(data, DeferredDatasetHandle):
287 return data.get(component=
"columns")
291 def _get_data(self, data):
292 """Retrieve dataframe necessary for calculation.
294 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
296 Returns dataframe upon which `self._func` can act.
298 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
300 if isinstance(data, pd.DataFrame):
305 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
308 if isinstance(data, ParquetTable)
and not is_multiLevel:
310 df = data.toDataFrame(columns=columns)
319 if isinstance(data, MultilevelParquetTable):
321 df = data.toDataFrame(columns=columns, droplevels=
False)
322 elif isinstance(data, DeferredDatasetHandle):
324 df = data.get(parameters={
"columns": columns})
332 def _setLevels(self, df):
333 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
334 df.columns = df.columns.droplevel(levelsToDrop)
337 def _dropna(self, vals):
343 vals = self.
_func_func(df)
345 vals = self.
failfail(df)
347 vals = self.
_dropna_dropna(vals)
352 """Computes difference between functor called on two different ParquetTable objects
354 return self(data1, **kwargs) - self(data2, **kwargs)
357 return pd.Series(np.full(len(df), np.nan), index=df.index)
361 """Full name of functor (suitable for figure labels)
363 return NotImplementedError
367 """Short name of functor (suitable for column name/dict key)
373 """Perform multiple calculations at once on a catalog
375 The role of a `CompositeFunctor` is to group together computations from
376 multiple functors. Instead of returning `pandas.Series` a
377 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
378 being the keys of `funcDict`.
380 The `columns` attribute of a `CompositeFunctor` is the union of all columns
381 in all the component functors.
383 A `CompositeFunctor` does not use a `_func` method itself; rather,
384 when a `CompositeFunctor` is called, all its columns are loaded
385 at once, and the resulting dataframe is passed to the `_func` method of each component
386 functor. This has the advantage of only doing I/O (reading from parquet file) once,
387 and works because each individual `_func` method of each component functor does not
388 care if there are *extra* columns in the dataframe being passed; only that it must contain
389 *at least* the `columns` it expects.
391 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
392 file specifying a collection of functors.
396 funcs : `dict` or `list`
397 Dictionary or list of functors. If a list, then it will be converted
398 into a dictonary according to the `.shortname` attribute of each functor.
405 if type(funcs) == dict:
408 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
410 self.
_filt_filt =
None
416 return self.
_filt_filt
421 for _, f
in self.
funcDictfuncDict.items():
423 self.
_filt_filt = filt
426 if isinstance(new, dict):
428 elif isinstance(new, CompositeFunctor):
431 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
439 return list(set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
448 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
456 """Apply the functor to the data table
460 data : `lsst.daf.butler.DeferredDatasetHandle`,
461 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
462 `lsst.pipe.tasks.parquetTable.ParquetTable`,
463 or `pandas.DataFrame`.
464 The table or a pointer to a table on disk from which columns can
470 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
476 if isinstance(data, MultilevelParquetTable):
478 df = data.toDataFrame(columns=columns, droplevels=
False)
479 elif isinstance(data, DeferredDatasetHandle):
481 df = data.get(parameters={
"columns": columns})
484 for k, f
in self.
funcDictfuncDict.items():
486 subdf = f._setLevels(
487 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
489 valDict[k] = f._func(subdf)
491 valDict[k] = f.fail(subdf)
494 if isinstance(data, DeferredDatasetHandle):
497 elif isinstance(data, pd.DataFrame):
504 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
507 valDf = pd.concat(valDict, axis=1)
509 print([(k, type(v))
for k, v
in valDict.items()])
512 if kwargs.get(
'dropna',
False):
513 valDf = valDf.dropna(how=
'any')
519 if renameRules
is None:
521 for old, new
in renameRules:
522 if col.startswith(old):
523 col = col.replace(old, new)
529 filename = os.path.expandvars(filename)
530 with open(filename)
as f:
531 translationDefinition = yaml.safe_load(f)
533 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
538 for func, val
in translationDefinition[
'funcs'].items():
541 if 'flag_rename_rules' in translationDefinition:
542 renameRules = translationDefinition[
'flag_rename_rules']
546 if 'calexpFlags' in translationDefinition:
547 for flag
in translationDefinition[
'calexpFlags']:
548 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
550 if 'refFlags' in translationDefinition:
551 for flag
in translationDefinition[
'refFlags']:
552 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
554 if 'forcedFlags' in translationDefinition:
555 for flag
in translationDefinition[
'forcedFlags']:
556 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
558 if 'flags' in translationDefinition:
559 for flag
in translationDefinition[
'flags']:
560 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
562 return cls(funcs, **kwargs)
566 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
568 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
572 df : pandas.DataFrame
573 Dataframe on which to evaluate expression.
579 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
580 val = df.eval(expr_new, truediv=
True)
582 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
583 val = df.eval(expr_new, truediv=
True)
588 """Arbitrary computation on a catalog
590 Column names (and thus the columns to be loaded from catalog) are found
591 by finding all words and trying to ignore all "math-y" words.
596 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
598 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
610 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
612 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
615 if not re.search(
'_instFlux$', c):
616 cols.append(f
'{c}_instFlux')
621 return list(set([c
for c
in cols
if c
not in not_a_col]))
628 """Get column with specified name
644 return df[self.
colcol]
648 """Return the value of the index for each object
651 columns = [
'coord_ra']
652 _defaultDataset =
'ref'
656 return pd.Series(df.index, index=df.index)
661 _allow_difference =
False
665 return pd.Series(df.index, index=df.index)
669 col =
'base_Footprint_nPix'
673 """Base class for coordinate column, in degrees
682 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
687 """Right Ascension, in degrees
693 super().
__init__(
'coord_ra', **kwargs)
696 return super().
__call__(catalog, **kwargs)
700 """Declination, in degrees
706 super().
__init__(
'coord_dec', **kwargs)
709 return super().
__call__(catalog, **kwargs)
713 """Compute the level 20 HtmIndex for the catalog.
728 def computePixel(row):
737 return self.
pixelatorpixelator.index(sphPoint.getVector())
739 return df.apply(computePixel, axis=1)
743 if not col.endswith(
'_instFlux'):
749 if not col.endswith(
'_instFluxErr'):
750 col +=
'_instFluxErr'
755 """Compute calibrated magnitude
757 Takes a `calib` argument, which returns the flux at mag=0
758 as `calib.getFluxMag0()`. If not provided, then the default
759 `fluxMag0` is 63095734448.0194, which is default for HSC.
760 This default should be removed in DM-21955
762 This calculation hides warnings about invalid values and dividing by zero.
764 As for all functors, a `dataset` and `filt` kwarg should be provided upon
765 initialization. Unlike the default `Functor`, however, the default dataset
766 for a `Mag` is `'meas'`, rather than `'ref'`.
771 Name of flux column from which to compute magnitude. Can be parseable
772 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
773 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
775 calib : `lsst.afw.image.calib.Calib` (optional)
776 Object that knows zero point.
778 _defaultDataset =
'meas'
783 if calib
is not None:
787 self.
fluxMag0fluxMag0 = 63095734448.0194
796 with np.warnings.catch_warnings():
797 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
798 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
799 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
803 return f
'mag_{self.col}'
807 """Compute calibrated magnitude uncertainty
809 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
814 calib : `lsst.afw.image.calib.Calib` (optional)
815 Object that knows zero point.
820 if self.
calibcalib
is not None:
827 return [self.
colcol, self.
colcol +
'Err']
830 with np.warnings.catch_warnings():
831 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
832 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
834 x = df[fluxErrCol] / df[fluxCol]
836 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
841 return super().name +
'_err'
849 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
853 _defaultDataset =
'meas'
855 """Functor to calculate magnitude difference"""
864 return [self.
col1col1, self.
col2col2]
867 with np.warnings.catch_warnings():
868 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
869 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
870 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
874 return f
'(mag_{self.col1} - mag_{self.col2})'
878 return f
'magDiff_{self.col1}_{self.col2}'
882 """Compute the color between two filters
884 Computes color by initializing two different `Mag`
885 functors based on the `col` and filters provided, and
886 then returning the difference.
888 This is enabled by the `_func` expecting a dataframe with a
889 multilevel column index, with both `'band'` and `'column'`,
890 instead of just `'column'`, which is the `Functor` default.
891 This is controlled by the `_dfLevels` attribute.
893 Also of note, the default dataset for `Color` is `forced_src'`,
894 whereas for `Mag` it is `'meas'`.
899 Name of flux column from which to compute; same as would be passed to
900 `lsst.pipe.tasks.functors.Mag`.
903 Filters from which to compute magnitude difference.
904 Color computed is `Mag(filt2) - Mag(filt1)`.
906 _defaultDataset =
'forced_src'
907 _dfLevels = (
'band',
'column')
913 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
917 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
918 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
931 mag2 = self.mag2._func(df[self.filt2])
932 mag1 = self.mag1._func(df[self.filt1])
937 return [self.
mag1mag1.col, self.
mag2mag2.col]
944 return f
'{self.filt2} - {self.filt1} ({self.col})'
948 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
952 """Main function of this subclass is to override the dropna=True
955 _allow_difference =
False
960 return super().
__call__(parq, dropna=
False, **kwargs)
964 _columns = [
"base_ClassificationExtendedness_value"]
965 _column =
"base_ClassificationExtendedness_value"
970 test = (x < 0.5).astype(int)
971 test = test.mask(mask, 2)
975 categories = [
'galaxy',
'star', self.
_null_label_null_label]
976 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
977 index=x.index, name=
'label')
979 label = label.astype(str)
984 _columns = [
'numStarFlags']
985 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
991 n = len(x.unique()) - 1
993 labels = [
'noStar',
'maybe',
'star']
994 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
995 index=x.index, name=
'label')
998 label = label.astype(str)
1004 name =
'Deconvolved Moments'
1005 shortname =
'deconvolvedMoments'
1006 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1007 "ext_shapeHSM_HsmSourceMoments_yy",
1008 "base_SdssShape_xx",
"base_SdssShape_yy",
1009 "ext_shapeHSM_HsmPsfMoments_xx",
1010 "ext_shapeHSM_HsmPsfMoments_yy")
1012 def _func(self, df):
1013 """Calculate deconvolved moments"""
1014 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1015 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1017 hsm = np.ones(len(df))*np.nan
1018 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1019 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1020 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1025 raise RuntimeError(
'No psf shape parameter found in catalog')
1027 return hsm.where(np.isfinite(hsm), sdss) - psf
1031 """Functor to calculate SDSS trace radius size for sources"""
1032 name =
"SDSS Trace Size"
1033 shortname =
'sdssTrace'
1034 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1036 def _func(self, df):
1037 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1042 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1043 name =
"PSF - SDSS Trace Size"
1044 shortname =
'psf_sdssTrace'
1045 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1046 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1048 def _func(self, df):
1049 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1050 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1051 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1056 """Functor to calculate HSM trace radius size for sources"""
1057 name =
'HSM Trace Size'
1058 shortname =
'hsmTrace'
1059 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1060 "ext_shapeHSM_HsmSourceMoments_yy")
1062 def _func(self, df):
1063 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1064 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1069 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1070 name =
'PSF - HSM Trace Size'
1071 shortname =
'psf_HsmTrace'
1072 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1073 "ext_shapeHSM_HsmSourceMoments_yy",
1074 "ext_shapeHSM_HsmPsfMoments_xx",
1075 "ext_shapeHSM_HsmPsfMoments_yy")
1077 def _func(self, df):
1078 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1079 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1080 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1081 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1082 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1087 name =
'HSM Psf FWHM'
1088 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1091 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1093 def _func(self, df):
1095 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1099 name =
"Distortion Ellipticity (e1)"
1100 shortname =
"Distortion"
1113 def _func(self, df):
1118 name =
"Ellipticity e2"
1130 def _func(self, df):
1131 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1146 def _func(self, df):
1147 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1151 """Computations using the stored localWcs.
1153 name =
"LocalWcsOperations"
1168 """Compute the distance on the sphere from x2, y1 to x1, y1.
1176 cd11 : `pandas.Series`
1177 [1, 1] element of the local Wcs affine transform.
1178 cd11 : `pandas.Series`
1179 [1, 1] element of the local Wcs affine transform.
1180 cd12 : `pandas.Series`
1181 [1, 2] element of the local Wcs affine transform.
1182 cd21 : `pandas.Series`
1183 [2, 1] element of the local Wcs affine transform.
1184 cd22 : `pandas.Series`
1185 [2, 2] element of the local Wcs affine transform.
1190 RA and dec conversion of x and y given the local Wcs. Returned
1191 units are in radians.
1194 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1197 """Compute the local pixel scale conversion.
1201 ra1 : `pandas.Series`
1202 Ra of the first coordinate in radians.
1203 dec1 : `pandas.Series`
1204 Dec of the first coordinate in radians.
1205 ra2 : `pandas.Series`
1206 Ra of the second coordinate in radians.
1207 dec2 : `pandas.Series`
1208 Dec of the second coordinate in radians.
1212 dist : `pandas.Series`
1213 Distance on the sphere in radians.
1215 deltaDec = dec2 - dec1
1217 return 2 * np.arcsin(
1219 np.sin(deltaDec / 2) ** 2
1220 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1223 """Compute the distance on the sphere from x2, y1 to x1, y1.
1227 x1 : `pandas.Series`
1229 y1 : `pandas.Series`
1231 x2 : `pandas.Series`
1233 y2 : `pandas.Series`
1235 cd11 : `pandas.Series`
1236 [1, 1] element of the local Wcs affine transform.
1237 cd11 : `pandas.Series`
1238 [1, 1] element of the local Wcs affine transform.
1239 cd12 : `pandas.Series`
1240 [1, 2] element of the local Wcs affine transform.
1241 cd21 : `pandas.Series`
1242 [2, 1] element of the local Wcs affine transform.
1243 cd22 : `pandas.Series`
1244 [2, 2] element of the local Wcs affine transform.
1248 Distance : `pandas.Series`
1249 Arcseconds per pixel at the location of the local WC
1251 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1252 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1258 """Compute the local pixel scale from the stored CDMatrix.
1270 """Compute the local pixel to scale conversion in arcseconds.
1274 cd11 : `pandas.Series`
1275 [1, 1] element of the local Wcs affine transform in radians.
1276 cd11 : `pandas.Series`
1277 [1, 1] element of the local Wcs affine transform in radians.
1278 cd12 : `pandas.Series`
1279 [1, 2] element of the local Wcs affine transform in radians.
1280 cd21 : `pandas.Series`
1281 [2, 1] element of the local Wcs affine transform in radians.
1282 cd22 : `pandas.Series`
1283 [2, 2] element of the local Wcs affine transform in radians.
1287 pixScale : `pandas.Series`
1288 Arcseconds per pixel at the location of the local WC
1290 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1292 def _func(self, df):
1300 """Convert a value in units pixels squared to units arcseconds squared.
1319 return f
"{self.col}_asArcseconds"
1323 return [self.
colcol,
1329 def _func(self, df):
1337 """Convert a value in units pixels to units arcseconds.
1356 return f
"{self.col}_asArcsecondsSq"
1360 return [self.
colcol,
1366 def _func(self, df):
1371 return df[self.
colcol] * pixScale * pixScale
1375 name =
'Reference Band'
1376 shortname =
'refBand'
1380 return [
"merge_measurement_i",
1381 "merge_measurement_r",
1382 "merge_measurement_z",
1383 "merge_measurement_y",
1384 "merge_measurement_g"]
1386 def _func(self, df):
1387 def getFilterAliasName(row):
1389 colName = row.idxmax()
1390 return colName.replace(
'merge_measurement_',
'')
1392 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1397 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1398 LOG_AB_FLUX_SCALE = 12.56
1399 FIVE_OVER_2LOG10 = 1.085736204758129569
1403 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1409 if calib
is not None:
1419 return [self.
colcol]
1423 return f
'mag_{self.col}'
1427 if np.abs(a) < np.abs(b):
1432 return np.abs(a) * np.sqrt(1. + q*q)
1438 with np.warnings.catch_warnings():
1439 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1440 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1441 return -2.5 * np.log10(dn/fluxMag0)
1444 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1445 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1449 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1454 def _func(self, df):
1463 def _func(self, df):
1465 return pd.Series(retArr, index=df.index)
1469 def _func(self, df):
1478 def _func(self, df):
1480 return pd.Series(retArr, index=df.index)
1484 """Base class for calibrating the specified instrument flux column using
1485 the local photometric calibration.
1490 Name of the instrument flux column.
1491 instFluxErrCol : `str`
1492 Name of the assocated error columns for ``instFluxCol``.
1493 photoCalibCol : `str`
1494 Name of local calibration column.
1495 photoCalibErrCol : `str`
1496 Error associated with ``photoCalibCol``
1506 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1521 """Convert instrument flux to nanojanskys.
1525 instFlux : `numpy.ndarray` or `pandas.Series`
1526 Array of instrument flux measurements
1527 localCalib : `numpy.ndarray` or `pandas.Series`
1528 Array of local photometric calibration estimates.
1532 calibFlux : `numpy.ndarray` or `pandas.Series`
1533 Array of calibrated flux measurements.
1535 return instFlux * localCalib
1538 """Convert instrument flux to nanojanskys.
1542 instFlux : `numpy.ndarray` or `pandas.Series`
1543 Array of instrument flux measurements
1544 instFluxErr : `numpy.ndarray` or `pandas.Series`
1545 Errors on associated ``instFlux`` values
1546 localCalib : `numpy.ndarray` or `pandas.Series`
1547 Array of local photometric calibration estimates.
1548 localCalibErr : `numpy.ndarray` or `pandas.Series`
1549 Errors on associated ``localCalib`` values
1553 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1554 Errors on calibrated flux measurements.
1556 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1559 """Convert instrument flux to nanojanskys.
1563 instFlux : `numpy.ndarray` or `pandas.Series`
1564 Array of instrument flux measurements
1565 localCalib : `numpy.ndarray` or `pandas.Series`
1566 Array of local photometric calibration estimates.
1570 calibMag : `numpy.ndarray` or `pandas.Series`
1571 Array of calibrated AB magnitudes.
1576 """Convert instrument flux err to nanojanskys.
1580 instFlux : `numpy.ndarray` or `pandas.Series`
1581 Array of instrument flux measurements
1582 instFluxErr : `numpy.ndarray` or `pandas.Series`
1583 Errors on associated ``instFlux`` values
1584 localCalib : `numpy.ndarray` or `pandas.Series`
1585 Array of local photometric calibration estimates.
1586 localCalibErr : `numpy.ndarray` or `pandas.Series`
1587 Errors on associated ``localCalib`` values
1591 calibMagErr: `numpy.ndarray` or `pandas.Series`
1592 Error on calibrated AB magnitudes.
1595 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1599 """Compute calibrated fluxes using the local calibration value.
1615 return f
'flux_{self.instFluxCol}'
1617 def _func(self, df):
1622 """Compute calibrated flux errors using the local calibration value.
1639 return f
'fluxErr_{self.instFluxCol}'
1641 def _func(self, df):
1647 """Compute calibrated AB magnitudes using the local calibration value.
1663 return f
'mag_{self.instFluxCol}'
1665 def _func(self, df):
1671 """Compute calibrated AB magnitude errors using the local calibration value.
1688 return f
'magErr_{self.instFluxCol}'
1690 def _func(self, df):
1698 """Compute absolute mean of dipole fluxes.
1707 LocalDipoleMeanFluxErr
1709 LocalDipoleDiffFluxErr
1739 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1741 def _func(self, df):
1747 """Compute the error on the absolute mean of dipole fluxes.
1756 LocalDipoleMeanFluxErr
1758 LocalDipoleDiffFluxErr
1772 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1774 def _func(self, df):
1783 """Compute the absolute difference of dipole fluxes.
1785 Value is (abs(pos) - abs(neg))
1794 LocalDipoleMeanFluxErr
1796 LocalDipoleDiffFluxErr
1807 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1809 def _func(self, df):
1815 """Compute the error on the absolute difference of dipole fluxes.
1824 LocalDipoleMeanFluxErr
1826 LocalDipoleDiffFluxErr
1840 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1842 def _func(self, df):
1851 """Base class for returning the ratio of 2 columns.
1853 Can be used to compute a Signal to Noise ratio for any input flux.
1858 Name of the column to use at the numerator in the ratio
1860 Name of the column to use as the denominator in the ratio.
1876 return f
'ratio_{self.numerator}_{self.denominator}'
1878 def _func(self, df):
1879 with np.warnings.catch_warnings():
1880 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1881 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)