24 from itertools
import product
29 import astropy.units
as u
32 from lsst.daf.butler
import DeferredDatasetHandle
33 from .parquetTable
import ParquetTable, MultilevelParquetTable
37 typeKey='functor', name=None):
38 """Initialize an object defined in a dictionary
40 The object needs to be importable as
41 f'{basePath}.{initDict[typeKey]}'
42 The positional and keyword arguments (if any) are contained in
43 "args" and "kwargs" entries in the dictionary, respectively.
44 This is used in `functors.CompositeFunctor.from_yaml` to initialize
45 a composite functor from a specification in a YAML file.
50 Dictionary describing object's initialization. Must contain
51 an entry keyed by ``typeKey`` that is the name of the object,
52 relative to ``basePath``.
54 Path relative to module in which ``initDict[typeKey]`` is defined.
56 Key of ``initDict`` that is the name of the object
57 (relative to `basePath`).
59 initDict = initDict.copy()
61 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
63 if 'args' in initDict:
64 args = initDict.pop(
'args')
65 if isinstance(args, str):
68 element = pythonType(*args, **initDict)
69 except Exception
as e:
70 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
71 raise type(e)(message, e.args)
76 """Define and execute a calculation on a ParquetTable
78 The `__call__` method accepts either a `ParquetTable` object or a
79 `DeferredDatasetHandle`, and returns the
80 result of the calculation as a single column. Each functor defines what
81 columns are needed for the calculation, and only these columns are read
82 from the `ParquetTable`.
84 The action of `__call__` consists of two steps: first, loading the
85 necessary columns from disk into memory as a `pandas.DataFrame` object;
86 and second, performing the computation on this dataframe and returning the
90 To define a new `Functor`, a subclass must define a `_func` method,
91 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
92 In addition, it must define the following attributes
94 * `_columns`: The columns necessary to perform the calculation
95 * `name`: A name appropriate for a figure axis label
96 * `shortname`: A name appropriate for use as a dictionary key
98 On initialization, a `Functor` should declare what band (`filt` kwarg)
99 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
100 applied to. This enables the `_get_data` method to extract the proper
101 columns from the parquet file. If not specified, the dataset will fall back
102 on the `_defaultDataset`attribute. If band is not specified and `dataset`
103 is anything other than `'ref'`, then an error will be raised when trying to
104 perform the calculation.
106 Originally, `Functor` was set up to expect
107 datasets formatted like the `deepCoadd_obj` dataset; that is, a
108 dataframe with a multi-level column index, with the levels of the
109 column index being `band`, `dataset`, and `column`.
110 It has since been generalized to apply to dataframes without mutli-level
111 indices and multi-level indices with just `dataset` and `column` levels.
112 In addition, the `_get_data` method that reads
113 the dataframe from the `ParquetTable` will return a dataframe with column
114 index levels defined by the `_dfLevels` attribute; by default, this is
117 The `_dfLevels` attributes should generally not need to
118 be changed, unless `_func` needs columns from multiple filters or datasets
119 to do the calculation.
120 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
121 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
122 it gets to have those levels in the column index.
127 Filter upon which to do the calculation
130 Dataset upon which to do the calculation
131 (e.g., 'ref', 'meas', 'forced_src').
135 _defaultDataset =
'ref'
136 _dfLevels = (
'column',)
137 _defaultNoDup =
False
139 def __init__(self, filt=None, dataset=None, noDup=None):
146 if self.
_noDup_noDup
is not None:
153 """Columns required to perform calculation
155 if not hasattr(self,
'_columns'):
156 raise NotImplementedError(
'Must define columns property or _columns attribute')
159 def _get_data_columnLevels(self, data, columnIndex=None):
160 """Gets the names of the column index levels
162 This should only be called in the context of a multilevel table.
163 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
164 and with the gen3 `DeferredDatasetHandle`.
168 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
170 columnnIndex (optional): pandas `Index` object
171 if not passed, then it is read from the `DeferredDatasetHandle`
173 if isinstance(data, DeferredDatasetHandle):
174 if columnIndex
is None:
175 columnIndex = data.get(component=
"columns")
176 if columnIndex
is not None:
177 return columnIndex.names
178 if isinstance(data, MultilevelParquetTable):
179 return data.columnLevels
181 raise TypeError(f
"Unknown type for data: {type(data)}!")
183 def _get_data_columnLevelNames(self, data, columnIndex=None):
184 """Gets the content of each of the column levels for a multilevel table
186 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
188 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
190 if isinstance(data, DeferredDatasetHandle):
191 if columnIndex
is None:
192 columnIndex = data.get(component=
"columns")
193 if columnIndex
is not None:
194 columnLevels = columnIndex.names
196 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
197 for i, level
in enumerate(columnLevels)
199 return columnLevelNames
200 if isinstance(data, MultilevelParquetTable):
201 return data.columnLevelNames
203 raise TypeError(f
"Unknown type for data: {type(data)}!")
205 def _colsFromDict(self, colDict, columnIndex=None):
206 """Converts dictionary column specficiation to a list of columns
208 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
213 for i, lev
in enumerate(columnLevels):
215 if isinstance(colDict[lev], str):
216 new_colDict[lev] = [colDict[lev]]
218 new_colDict[lev] = colDict[lev]
220 new_colDict[lev] = columnIndex.levels[i]
222 levelCols = [new_colDict[lev]
for lev
in columnLevels]
223 cols = product(*levelCols)
227 """Returns columns needed by functor from multilevel dataset
229 To access tables with multilevel column structure, the `MultilevelParquetTable`
230 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
235 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
237 columnIndex (optional): pandas `Index` object
238 either passed or read in from `DeferredDatasetHandle`.
241 If true, then return a list of tuples rather than the column dictionary
242 specification. This is set to `True` by `CompositeFunctor` in order to be able to
243 combine columns from the various component functors.
246 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
247 columnIndex = data.get(component=
"columns")
252 columnDict = {
'column': self.
columnscolumns,
253 'dataset': self.
datasetdataset}
254 if self.
filtfilt
is None:
256 if "band" in columnLevels:
257 if self.
datasetdataset ==
"ref":
258 columnDict[
"band"] = columnLevelNames[
"band"][0]
260 raise ValueError(f
"'filt' not set for functor {self.name}"
261 f
"(dataset {self.dataset}) "
263 "contains multiple filters in column index. "
264 "Set 'filt' or set 'dataset' to 'ref'.")
266 columnDict[
'band'] = self.
filtfilt
268 if isinstance(data, MultilevelParquetTable):
269 return data._colsFromDict(columnDict)
270 elif isinstance(data, DeferredDatasetHandle):
272 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
276 def _func(self, df, dropna=True):
277 raise NotImplementedError(
'Must define calculation on dataframe')
279 def _get_columnIndex(self, data):
280 """Return columnIndex
283 if isinstance(data, DeferredDatasetHandle):
284 return data.get(component=
"columns")
288 def _get_data(self, data):
289 """Retrieve dataframe necessary for calculation.
291 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
293 Returns dataframe upon which `self._func` can act.
295 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
297 if isinstance(data, pd.DataFrame):
302 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
305 if isinstance(data, ParquetTable)
and not is_multiLevel:
307 df = data.toDataFrame(columns=columns)
316 if isinstance(data, MultilevelParquetTable):
318 df = data.toDataFrame(columns=columns, droplevels=
False)
319 elif isinstance(data, DeferredDatasetHandle):
321 df = data.get(parameters={
"columns": columns})
329 def _setLevels(self, df):
330 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
331 df.columns = df.columns.droplevel(levelsToDrop)
334 def _dropna(self, vals):
340 vals = self.
_func_func(df)
342 vals = self.
failfail(df)
344 vals = self.
_dropna_dropna(vals)
349 """Computes difference between functor called on two different ParquetTable objects
351 return self(data1, **kwargs) - self(data2, **kwargs)
354 return pd.Series(np.full(len(df), np.nan), index=df.index)
358 """Full name of functor (suitable for figure labels)
360 return NotImplementedError
364 """Short name of functor (suitable for column name/dict key)
370 """Perform multiple calculations at once on a catalog
372 The role of a `CompositeFunctor` is to group together computations from
373 multiple functors. Instead of returning `pandas.Series` a
374 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
375 being the keys of `funcDict`.
377 The `columns` attribute of a `CompositeFunctor` is the union of all columns
378 in all the component functors.
380 A `CompositeFunctor` does not use a `_func` method itself; rather,
381 when a `CompositeFunctor` is called, all its columns are loaded
382 at once, and the resulting dataframe is passed to the `_func` method of each component
383 functor. This has the advantage of only doing I/O (reading from parquet file) once,
384 and works because each individual `_func` method of each component functor does not
385 care if there are *extra* columns in the dataframe being passed; only that it must contain
386 *at least* the `columns` it expects.
388 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
389 file specifying a collection of functors.
393 funcs : `dict` or `list`
394 Dictionary or list of functors. If a list, then it will be converted
395 into a dictonary according to the `.shortname` attribute of each functor.
402 if type(funcs) == dict:
405 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
407 self.
_filt_filt =
None
413 return self.
_filt_filt
418 for _, f
in self.
funcDictfuncDict.items():
420 self.
_filt_filt = filt
423 if isinstance(new, dict):
425 elif isinstance(new, CompositeFunctor):
428 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
436 return list(set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
445 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
453 """Apply the functor to the data table
457 data : `lsst.daf.butler.DeferredDatasetHandle`,
458 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
459 `lsst.pipe.tasks.parquetTable.ParquetTable`,
460 or `pandas.DataFrame`.
461 The table or a pointer to a table on disk from which columns can
467 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
473 if isinstance(data, MultilevelParquetTable):
475 df = data.toDataFrame(columns=columns, droplevels=
False)
476 elif isinstance(data, DeferredDatasetHandle):
478 df = data.get(parameters={
"columns": columns})
481 for k, f
in self.
funcDictfuncDict.items():
483 subdf = f._setLevels(
484 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
486 valDict[k] = f._func(subdf)
488 valDict[k] = f.fail(subdf)
491 if isinstance(data, DeferredDatasetHandle):
494 elif isinstance(data, pd.DataFrame):
501 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
504 valDf = pd.concat(valDict, axis=1)
506 print([(k, type(v))
for k, v
in valDict.items()])
509 if kwargs.get(
'dropna',
False):
510 valDf = valDf.dropna(how=
'any')
516 if renameRules
is None:
518 for old, new
in renameRules:
519 if col.startswith(old):
520 col = col.replace(old, new)
526 filename = os.path.expandvars(filename)
527 with open(filename)
as f:
528 translationDefinition = yaml.safe_load(f)
530 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
535 for func, val
in translationDefinition[
'funcs'].items():
538 if 'flag_rename_rules' in translationDefinition:
539 renameRules = translationDefinition[
'flag_rename_rules']
543 if 'calexpFlags' in translationDefinition:
544 for flag
in translationDefinition[
'calexpFlags']:
545 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
547 if 'refFlags' in translationDefinition:
548 for flag
in translationDefinition[
'refFlags']:
549 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
551 if 'forcedFlags' in translationDefinition:
552 for flag
in translationDefinition[
'forcedFlags']:
553 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
555 if 'flags' in translationDefinition:
556 for flag
in translationDefinition[
'flags']:
557 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
559 return cls(funcs, **kwargs)
563 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
565 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
569 df : pandas.DataFrame
570 Dataframe on which to evaluate expression.
576 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
577 val = df.eval(expr_new, truediv=
True)
579 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
580 val = df.eval(expr_new, truediv=
True)
585 """Arbitrary computation on a catalog
587 Column names (and thus the columns to be loaded from catalog) are found
588 by finding all words and trying to ignore all "math-y" words.
593 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
595 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
607 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
609 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
612 if not re.search(
'_instFlux$', c):
613 cols.append(f
'{c}_instFlux')
618 return list(set([c
for c
in cols
if c
not in not_a_col]))
625 """Get column with specified name
641 return df[self.
colcol]
645 """Return the value of the index for each object
648 columns = [
'coord_ra']
649 _defaultDataset =
'ref'
653 return pd.Series(df.index, index=df.index)
658 _allow_difference =
False
662 return pd.Series(df.index, index=df.index)
666 col =
'base_Footprint_nPix'
670 """Base class for coordinate column, in degrees
679 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
684 """Right Ascension, in degrees
690 super().
__init__(
'coord_ra', **kwargs)
693 return super().
__call__(catalog, **kwargs)
697 """Declination, in degrees
703 super().
__init__(
'coord_dec', **kwargs)
706 return super().
__call__(catalog, **kwargs)
710 if not col.endswith(
'_instFlux'):
716 if not col.endswith(
'_instFluxErr'):
717 col +=
'_instFluxErr'
722 """Compute calibrated magnitude
724 Takes a `calib` argument, which returns the flux at mag=0
725 as `calib.getFluxMag0()`. If not provided, then the default
726 `fluxMag0` is 63095734448.0194, which is default for HSC.
727 This default should be removed in DM-21955
729 This calculation hides warnings about invalid values and dividing by zero.
731 As for all functors, a `dataset` and `filt` kwarg should be provided upon
732 initialization. Unlike the default `Functor`, however, the default dataset
733 for a `Mag` is `'meas'`, rather than `'ref'`.
738 Name of flux column from which to compute magnitude. Can be parseable
739 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
740 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
742 calib : `lsst.afw.image.calib.Calib` (optional)
743 Object that knows zero point.
745 _defaultDataset =
'meas'
750 if calib
is not None:
754 self.
fluxMag0fluxMag0 = 63095734448.0194
763 with np.warnings.catch_warnings():
764 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
765 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
766 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
770 return f
'mag_{self.col}'
774 """Compute calibrated magnitude uncertainty
776 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
781 calib : `lsst.afw.image.calib.Calib` (optional)
782 Object that knows zero point.
787 if self.
calibcalib
is not None:
794 return [self.
colcol, self.
colcol +
'Err']
797 with np.warnings.catch_warnings():
798 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
799 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
801 x = df[fluxErrCol] / df[fluxCol]
803 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
808 return super().name +
'_err'
816 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
820 _defaultDataset =
'meas'
822 """Functor to calculate magnitude difference"""
831 return [self.
col1col1, self.
col2col2]
834 with np.warnings.catch_warnings():
835 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
836 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
837 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
841 return f
'(mag_{self.col1} - mag_{self.col2})'
845 return f
'magDiff_{self.col1}_{self.col2}'
849 """Compute the color between two filters
851 Computes color by initializing two different `Mag`
852 functors based on the `col` and filters provided, and
853 then returning the difference.
855 This is enabled by the `_func` expecting a dataframe with a
856 multilevel column index, with both `'band'` and `'column'`,
857 instead of just `'column'`, which is the `Functor` default.
858 This is controlled by the `_dfLevels` attribute.
860 Also of note, the default dataset for `Color` is `forced_src'`,
861 whereas for `Mag` it is `'meas'`.
866 Name of flux column from which to compute; same as would be passed to
867 `lsst.pipe.tasks.functors.Mag`.
870 Filters from which to compute magnitude difference.
871 Color computed is `Mag(filt2) - Mag(filt1)`.
873 _defaultDataset =
'forced_src'
874 _dfLevels = (
'band',
'column')
880 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
884 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
885 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
898 mag2 = self.mag2._func(df[self.filt2])
899 mag1 = self.mag1._func(df[self.filt1])
904 return [self.
mag1mag1.col, self.
mag2mag2.col]
911 return f
'{self.filt2} - {self.filt1} ({self.col})'
915 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
919 """Main function of this subclass is to override the dropna=True
922 _allow_difference =
False
927 return super().
__call__(parq, dropna=
False, **kwargs)
931 _columns = [
"base_ClassificationExtendedness_value"]
932 _column =
"base_ClassificationExtendedness_value"
937 test = (x < 0.5).astype(int)
938 test = test.mask(mask, 2)
942 categories = [
'galaxy',
'star', self.
_null_label_null_label]
943 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
944 index=x.index, name=
'label')
946 label = label.astype(str)
951 _columns = [
'numStarFlags']
952 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
958 n = len(x.unique()) - 1
960 labels = [
'noStar',
'maybe',
'star']
961 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
962 index=x.index, name=
'label')
965 label = label.astype(str)
971 name =
'Deconvolved Moments'
972 shortname =
'deconvolvedMoments'
973 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
974 "ext_shapeHSM_HsmSourceMoments_yy",
975 "base_SdssShape_xx",
"base_SdssShape_yy",
976 "ext_shapeHSM_HsmPsfMoments_xx",
977 "ext_shapeHSM_HsmPsfMoments_yy")
980 """Calculate deconvolved moments"""
981 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
982 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
984 hsm = np.ones(len(df))*np.nan
985 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
986 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
987 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
992 raise RuntimeError(
'No psf shape parameter found in catalog')
994 return hsm.where(np.isfinite(hsm), sdss) - psf
998 """Functor to calculate SDSS trace radius size for sources"""
999 name =
"SDSS Trace Size"
1000 shortname =
'sdssTrace'
1001 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1003 def _func(self, df):
1004 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1009 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1010 name =
"PSF - SDSS Trace Size"
1011 shortname =
'psf_sdssTrace'
1012 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1013 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1015 def _func(self, df):
1016 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1017 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1018 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1023 """Functor to calculate HSM trace radius size for sources"""
1024 name =
'HSM Trace Size'
1025 shortname =
'hsmTrace'
1026 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1027 "ext_shapeHSM_HsmSourceMoments_yy")
1029 def _func(self, df):
1030 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1031 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1036 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1037 name =
'PSF - HSM Trace Size'
1038 shortname =
'psf_HsmTrace'
1039 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1040 "ext_shapeHSM_HsmSourceMoments_yy",
1041 "ext_shapeHSM_HsmPsfMoments_xx",
1042 "ext_shapeHSM_HsmPsfMoments_yy")
1044 def _func(self, df):
1045 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1046 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1047 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1048 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1049 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1054 name =
'HSM Psf FWHM'
1055 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1058 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1060 def _func(self, df):
1062 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1066 name =
"Distortion Ellipticity (e1)"
1067 shortname =
"Distortion"
1080 def _func(self, df):
1085 name =
"Ellipticity e2"
1097 def _func(self, df):
1098 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1113 def _func(self, df):
1114 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1118 """Computations using the stored localWcs.
1120 name =
"LocalWcsOperations"
1135 """Compute the distance on the sphere from x2, y1 to x1, y1.
1143 cd11 : `pandas.Series`
1144 [1, 1] element of the local Wcs affine transform.
1145 cd11 : `pandas.Series`
1146 [1, 1] element of the local Wcs affine transform.
1147 cd12 : `pandas.Series`
1148 [1, 2] element of the local Wcs affine transform.
1149 cd21 : `pandas.Series`
1150 [2, 1] element of the local Wcs affine transform.
1151 cd22 : `pandas.Series`
1152 [2, 2] element of the local Wcs affine transform.
1157 RA and dec conversion of x and y given the local Wcs. Returned
1158 units are in radians.
1161 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1164 """Compute the local pixel scale conversion.
1168 ra1 : `pandas.Series`
1169 Ra of the first coordinate in radians.
1170 dec1 : `pandas.Series`
1171 Dec of the first coordinate in radians.
1172 ra2 : `pandas.Series`
1173 Ra of the second coordinate in radians.
1174 dec2 : `pandas.Series`
1175 Dec of the second coordinate in radians.
1179 dist : `pandas.Series`
1180 Distance on the sphere in radians.
1182 deltaDec = dec2 - dec1
1184 return 2 * np.arcsin(
1186 np.sin(deltaDec / 2) ** 2
1187 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1190 """Compute the distance on the sphere from x2, y1 to x1, y1.
1194 x1 : `pandas.Series`
1196 y1 : `pandas.Series`
1198 x2 : `pandas.Series`
1200 y2 : `pandas.Series`
1202 cd11 : `pandas.Series`
1203 [1, 1] element of the local Wcs affine transform.
1204 cd11 : `pandas.Series`
1205 [1, 1] element of the local Wcs affine transform.
1206 cd12 : `pandas.Series`
1207 [1, 2] element of the local Wcs affine transform.
1208 cd21 : `pandas.Series`
1209 [2, 1] element of the local Wcs affine transform.
1210 cd22 : `pandas.Series`
1211 [2, 2] element of the local Wcs affine transform.
1215 Distance : `pandas.Series`
1216 Arcseconds per pixel at the location of the local WC
1218 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1219 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1225 """Compute the local pixel scale from the stored CDMatrix.
1237 """Compute the local pixel to scale conversion in arcseconds.
1241 cd11 : `pandas.Series`
1242 [1, 1] element of the local Wcs affine transform in radians.
1243 cd11 : `pandas.Series`
1244 [1, 1] element of the local Wcs affine transform in radians.
1245 cd12 : `pandas.Series`
1246 [1, 2] element of the local Wcs affine transform in radians.
1247 cd21 : `pandas.Series`
1248 [2, 1] element of the local Wcs affine transform in radians.
1249 cd22 : `pandas.Series`
1250 [2, 2] element of the local Wcs affine transform in radians.
1254 pixScale : `pandas.Series`
1255 Arcseconds per pixel at the location of the local WC
1257 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1259 def _func(self, df):
1267 """Convert a value in units pixels squared to units arcseconds squared.
1286 return f
"{self.col}_asArcseconds"
1290 return [self.
colcol,
1296 def _func(self, df):
1304 """Convert a value in units pixels to units arcseconds.
1323 return f
"{self.col}_asArcsecondsSq"
1327 return [self.
colcol,
1333 def _func(self, df):
1338 return df[self.
colcol] * pixScale * pixScale
1342 name =
'Reference Band'
1343 shortname =
'refBand'
1347 return [
"merge_measurement_i",
1348 "merge_measurement_r",
1349 "merge_measurement_z",
1350 "merge_measurement_y",
1351 "merge_measurement_g"]
1353 def _func(self, df):
1354 def getFilterAliasName(row):
1356 colName = row.idxmax()
1357 return colName.replace(
'merge_measurement_',
'')
1359 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1364 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1365 LOG_AB_FLUX_SCALE = 12.56
1366 FIVE_OVER_2LOG10 = 1.085736204758129569
1370 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1376 if calib
is not None:
1386 return [self.
colcol]
1390 return f
'mag_{self.col}'
1394 if np.abs(a) < np.abs(b):
1399 return np.abs(a) * np.sqrt(1. + q*q)
1405 with np.warnings.catch_warnings():
1406 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1407 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1408 return -2.5 * np.log10(dn/fluxMag0)
1411 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1412 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1416 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1421 def _func(self, df):
1430 def _func(self, df):
1432 return pd.Series(retArr, index=df.index)
1436 def _func(self, df):
1445 def _func(self, df):
1447 return pd.Series(retArr, index=df.index)
1451 """Base class for calibrating the specified instrument flux column using
1452 the local photometric calibration.
1457 Name of the instrument flux column.
1458 instFluxErrCol : `str`
1459 Name of the assocated error columns for ``instFluxCol``.
1460 photoCalibCol : `str`
1461 Name of local calibration column.
1462 photoCalibErrCol : `str`
1463 Error associated with ``photoCalibCol``
1473 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1488 """Convert instrument flux to nanojanskys.
1492 instFlux : `numpy.ndarray` or `pandas.Series`
1493 Array of instrument flux measurements
1494 localCalib : `numpy.ndarray` or `pandas.Series`
1495 Array of local photometric calibration estimates.
1499 calibFlux : `numpy.ndarray` or `pandas.Series`
1500 Array of calibrated flux measurements.
1502 return instFlux * localCalib
1505 """Convert instrument flux to nanojanskys.
1509 instFlux : `numpy.ndarray` or `pandas.Series`
1510 Array of instrument flux measurements
1511 instFluxErr : `numpy.ndarray` or `pandas.Series`
1512 Errors on associated ``instFlux`` values
1513 localCalib : `numpy.ndarray` or `pandas.Series`
1514 Array of local photometric calibration estimates.
1515 localCalibErr : `numpy.ndarray` or `pandas.Series`
1516 Errors on associated ``localCalib`` values
1520 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1521 Errors on calibrated flux measurements.
1523 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1526 """Convert instrument flux to nanojanskys.
1530 instFlux : `numpy.ndarray` or `pandas.Series`
1531 Array of instrument flux measurements
1532 localCalib : `numpy.ndarray` or `pandas.Series`
1533 Array of local photometric calibration estimates.
1537 calibMag : `numpy.ndarray` or `pandas.Series`
1538 Array of calibrated AB magnitudes.
1543 """Convert instrument flux err to nanojanskys.
1547 instFlux : `numpy.ndarray` or `pandas.Series`
1548 Array of instrument flux measurements
1549 instFluxErr : `numpy.ndarray` or `pandas.Series`
1550 Errors on associated ``instFlux`` values
1551 localCalib : `numpy.ndarray` or `pandas.Series`
1552 Array of local photometric calibration estimates.
1553 localCalibErr : `numpy.ndarray` or `pandas.Series`
1554 Errors on associated ``localCalib`` values
1558 calibMagErr: `numpy.ndarray` or `pandas.Series`
1559 Error on calibrated AB magnitudes.
1562 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1566 """Compute calibrated fluxes using the local calibration value.
1582 return f
'flux_{self.instFluxCol}'
1584 def _func(self, df):
1589 """Compute calibrated flux errors using the local calibration value.
1606 return f
'fluxErr_{self.instFluxCol}'
1608 def _func(self, df):
1614 """Compute calibrated AB magnitudes using the local calibration value.
1630 return f
'mag_{self.instFluxCol}'
1632 def _func(self, df):
1638 """Compute calibrated AB magnitude errors using the local calibration value.
1655 return f
'magErr_{self.instFluxCol}'
1657 def _func(self, df):
1665 """Compute absolute mean of dipole fluxes.
1674 LocalDipoleMeanFluxErr
1676 LocalDipoleDiffFluxErr
1706 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1708 def _func(self, df):
1714 """Compute the error on the absolute mean of dipole fluxes.
1723 LocalDipoleMeanFluxErr
1725 LocalDipoleDiffFluxErr
1739 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1741 def _func(self, df):
1750 """Compute the absolute difference of dipole fluxes.
1752 Value is (abs(pos) - abs(neg))
1761 LocalDipoleMeanFluxErr
1763 LocalDipoleDiffFluxErr
1774 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1776 def _func(self, df):
1782 """Compute the error on the absolute difference of dipole fluxes.
1791 LocalDipoleMeanFluxErr
1793 LocalDipoleDiffFluxErr
1807 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1809 def _func(self, df):
1818 """Base class for returning the ratio of 2 columns.
1820 Can be used to compute a Signal to Noise ratio for any input flux.
1825 Name of the column to use at the numerator in the ratio
1827 Name of the column to use as the denominator in the ratio.
1843 return f
'ratio_{self.numerator}_{self.denominator}'
1845 def _func(self, df):
1846 with np.warnings.catch_warnings():
1847 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1848 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)