24 from itertools
import product
29 import astropy.units
as u
32 from lsst.daf.butler
import DeferredDatasetHandle
33 from .parquetTable
import ParquetTable, MultilevelParquetTable
37 typeKey='functor', name=None):
38 """Initialize an object defined in a dictionary
40 The object needs to be importable as
41 f'{basePath}.{initDict[typeKey]}'
42 The positional and keyword arguments (if any) are contained in
43 "args" and "kwargs" entries in the dictionary, respectively.
44 This is used in `functors.CompositeFunctor.from_yaml` to initialize
45 a composite functor from a specification in a YAML file.
50 Dictionary describing object's initialization. Must contain
51 an entry keyed by ``typeKey`` that is the name of the object,
52 relative to ``basePath``.
54 Path relative to module in which ``initDict[typeKey]`` is defined.
56 Key of ``initDict`` that is the name of the object
57 (relative to `basePath`).
59 initDict = initDict.copy()
61 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
63 if 'args' in initDict:
64 args = initDict.pop(
'args')
65 if isinstance(args, str):
68 element = pythonType(*args, **initDict)
69 except Exception
as e:
70 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
71 raise type(e)(message, e.args)
76 """Define and execute a calculation on a ParquetTable
78 The `__call__` method accepts either a `ParquetTable` object or a
79 `DeferredDatasetHandle`, and returns the
80 result of the calculation as a single column. Each functor defines what
81 columns are needed for the calculation, and only these columns are read
82 from the `ParquetTable`.
84 The action of `__call__` consists of two steps: first, loading the
85 necessary columns from disk into memory as a `pandas.DataFrame` object;
86 and second, performing the computation on this dataframe and returning the
90 To define a new `Functor`, a subclass must define a `_func` method,
91 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
92 In addition, it must define the following attributes
94 * `_columns`: The columns necessary to perform the calculation
95 * `name`: A name appropriate for a figure axis label
96 * `shortname`: A name appropriate for use as a dictionary key
98 On initialization, a `Functor` should declare what band (`filt` kwarg)
99 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
100 applied to. This enables the `_get_data` method to extract the proper
101 columns from the parquet file. If not specified, the dataset will fall back
102 on the `_defaultDataset`attribute. If band is not specified and `dataset`
103 is anything other than `'ref'`, then an error will be raised when trying to
104 perform the calculation.
106 As currently implemented, `Functor` is only set up to expect a
107 dataset of the format of the `deepCoadd_obj` dataset; that is, a
108 dataframe with a multi-level column index,
109 with the levels of the column index being `band`,
110 `dataset`, and `column`. This is defined in the `_columnLevels` attribute,
111 as well as being implicit in the role of the `filt` and `dataset` attributes
112 defined at initialization. In addition, the `_get_data` method that reads
113 the dataframe from the `ParquetTable` will return a dataframe with column
114 index levels defined by the `_dfLevels` attribute; by default, this is
117 The `_columnLevels` and `_dfLevels` attributes should generally not need to
118 be changed, unless `_func` needs columns from multiple filters or datasets
119 to do the calculation.
120 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
121 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
122 it gets to have those levels in the column index.
127 Filter upon which to do the calculation
130 Dataset upon which to do the calculation
131 (e.g., 'ref', 'meas', 'forced_src').
135 _defaultDataset =
'ref'
136 _columnLevels = (
'band',
'dataset',
'column')
137 _dfLevels = (
'column',)
138 _defaultNoDup =
False
140 def __init__(self, filt=None, dataset=None, noDup=None):
147 if self.
_noDup_noDup
is not None:
154 """Columns required to perform calculation
156 if not hasattr(self,
'_columns'):
157 raise NotImplementedError(
'Must define columns property or _columns attribute')
160 def _get_data_columnLevels(self, data, columnIndex=None):
161 """Gets the names of the column index levels
163 This should only be called in the context of a multilevel table.
164 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
165 and with the gen3 `DeferredDatasetHandle`.
169 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
171 columnnIndex (optional): pandas `Index` object
172 if not passed, then it is read from the `DeferredDatasetHandle`
174 if isinstance(data, DeferredDatasetHandle):
175 if columnIndex
is None:
176 columnIndex = data.get(component=
"columns")
177 if columnIndex
is not None:
178 return columnIndex.names
179 if isinstance(data, MultilevelParquetTable):
180 return data.columnLevels
182 raise TypeError(f
"Unknown type for data: {type(data)}!")
184 def _get_data_columnLevelNames(self, data, columnIndex=None):
185 """Gets the content of each of the column levels for a multilevel table
187 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
189 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
191 if isinstance(data, DeferredDatasetHandle):
192 if columnIndex
is None:
193 columnIndex = data.get(component=
"columns")
194 if columnIndex
is not None:
195 columnLevels = columnIndex.names
197 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
198 for i, level
in enumerate(columnLevels)
200 return columnLevelNames
201 if isinstance(data, MultilevelParquetTable):
202 return data.columnLevelNames
204 raise TypeError(f
"Unknown type for data: {type(data)}!")
206 def _colsFromDict(self, colDict, columnIndex=None):
207 """Converts dictionary column specficiation to a list of columns
209 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
214 for i, lev
in enumerate(columnLevels):
216 if isinstance(colDict[lev], str):
217 new_colDict[lev] = [colDict[lev]]
219 new_colDict[lev] = colDict[lev]
221 new_colDict[lev] = columnIndex.levels[i]
223 levelCols = [new_colDict[lev]
for lev
in columnLevels]
224 cols = product(*levelCols)
228 """Returns columns needed by functor from multilevel dataset
230 To access tables with multilevel column structure, the `MultilevelParquetTable`
231 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
236 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
238 columnIndex (optional): pandas `Index` object
239 either passed or read in from `DeferredDatasetHandle`.
242 If true, then return a list of tuples rather than the column dictionary
243 specification. This is set to `True` by `CompositeFunctor` in order to be able to
244 combine columns from the various component functors.
247 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
248 columnIndex = data.get(component=
"columns")
253 if not set(columnLevels) == set(self.
_columnLevels_columnLevels):
255 "ParquetTable does not have the expected column levels. "
256 f
"Got {columnLevels}; expected {self._columnLevels}."
259 columnDict = {
'column': self.
columnscolumns,
260 'dataset': self.
datasetdataset}
261 if self.
filtfilt
is None:
263 if "band" in columnLevels:
264 if self.
datasetdataset ==
"ref":
265 columnDict[
"band"] = columnLevelNames[
"band"][0]
267 raise ValueError(f
"'filt' not set for functor {self.name}"
268 f
"(dataset {self.dataset}) "
270 "contains multiple filters in column index. "
271 "Set 'filt' or set 'dataset' to 'ref'.")
273 columnDict[
'band'] = self.
filtfilt
275 if isinstance(data, MultilevelParquetTable):
276 return data._colsFromDict(columnDict)
277 elif isinstance(data, DeferredDatasetHandle):
279 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
283 def _func(self, df, dropna=True):
284 raise NotImplementedError(
'Must define calculation on dataframe')
286 def _get_columnIndex(self, data):
287 """Return columnIndex
290 if isinstance(data, DeferredDatasetHandle):
291 return data.get(component=
"columns")
295 def _get_data(self, data):
296 """Retrieve dataframe necessary for calculation.
298 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
300 Returns dataframe upon which `self._func` can act.
302 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
304 if isinstance(data, pd.DataFrame):
309 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
312 if isinstance(data, ParquetTable)
and not is_multiLevel:
314 df = data.toDataFrame(columns=columns)
323 if isinstance(data, MultilevelParquetTable):
325 df = data.toDataFrame(columns=columns, droplevels=
False)
326 elif isinstance(data, DeferredDatasetHandle):
328 df = data.get(parameters={
"columns": columns})
336 def _setLevels(self, df):
337 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
338 df.columns = df.columns.droplevel(levelsToDrop)
341 def _dropna(self, vals):
347 vals = self.
_func_func(df)
349 vals = self.
failfail(df)
351 vals = self.
_dropna_dropna(vals)
356 """Computes difference between functor called on two different ParquetTable objects
358 return self(data1, **kwargs) - self(data2, **kwargs)
361 return pd.Series(np.full(len(df), np.nan), index=df.index)
365 """Full name of functor (suitable for figure labels)
367 return NotImplementedError
371 """Short name of functor (suitable for column name/dict key)
377 """Perform multiple calculations at once on a catalog
379 The role of a `CompositeFunctor` is to group together computations from
380 multiple functors. Instead of returning `pandas.Series` a
381 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
382 being the keys of `funcDict`.
384 The `columns` attribute of a `CompositeFunctor` is the union of all columns
385 in all the component functors.
387 A `CompositeFunctor` does not use a `_func` method itself; rather,
388 when a `CompositeFunctor` is called, all its columns are loaded
389 at once, and the resulting dataframe is passed to the `_func` method of each component
390 functor. This has the advantage of only doing I/O (reading from parquet file) once,
391 and works because each individual `_func` method of each component functor does not
392 care if there are *extra* columns in the dataframe being passed; only that it must contain
393 *at least* the `columns` it expects.
395 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
396 file specifying a collection of functors.
400 funcs : `dict` or `list`
401 Dictionary or list of functors. If a list, then it will be converted
402 into a dictonary according to the `.shortname` attribute of each functor.
409 if type(funcs) == dict:
412 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
414 self.
_filt_filt =
None
420 return self.
_filt_filt
425 for _, f
in self.
funcDictfuncDict.items():
427 self.
_filt_filt = filt
430 if isinstance(new, dict):
432 elif isinstance(new, CompositeFunctor):
435 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
443 return list(set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
452 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
460 """Apply the functor to the data table
464 data : `lsst.daf.butler.DeferredDatasetHandle`,
465 `lsst.pipe.tasks.parquetTable.MultilevelParquetTable`,
466 `lsst.pipe.tasks.parquetTable.ParquetTable`,
467 or `pandas.DataFrame`.
468 The table or a pointer to a table on disk from which columns can
474 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
480 if isinstance(data, MultilevelParquetTable):
482 df = data.toDataFrame(columns=columns, droplevels=
False)
483 elif isinstance(data, DeferredDatasetHandle):
485 df = data.get(parameters={
"columns": columns})
488 for k, f
in self.
funcDictfuncDict.items():
490 subdf = f._setLevels(
491 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
493 valDict[k] = f._func(subdf)
495 valDict[k] = f.fail(subdf)
498 if isinstance(data, DeferredDatasetHandle):
501 elif isinstance(data, pd.DataFrame):
508 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
511 valDf = pd.concat(valDict, axis=1)
513 print([(k, type(v))
for k, v
in valDict.items()])
516 if kwargs.get(
'dropna',
False):
517 valDf = valDf.dropna(how=
'any')
523 if renameRules
is None:
525 for old, new
in renameRules:
526 if col.startswith(old):
527 col = col.replace(old, new)
533 filename = os.path.expandvars(filename)
534 with open(filename)
as f:
535 translationDefinition = yaml.safe_load(f)
537 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
542 for func, val
in translationDefinition[
'funcs'].items():
545 if 'flag_rename_rules' in translationDefinition:
546 renameRules = translationDefinition[
'flag_rename_rules']
550 if 'refFlags' in translationDefinition:
551 for flag
in translationDefinition[
'refFlags']:
552 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
554 if 'flags' in translationDefinition:
555 for flag
in translationDefinition[
'flags']:
556 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
558 return cls(funcs, **kwargs)
562 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
564 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
568 df : pandas.DataFrame
569 Dataframe on which to evaluate expression.
575 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
576 val = df.eval(expr_new, truediv=
True)
578 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
579 val = df.eval(expr_new, truediv=
True)
584 """Arbitrary computation on a catalog
586 Column names (and thus the columns to be loaded from catalog) are found
587 by finding all words and trying to ignore all "math-y" words.
592 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
594 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
606 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
608 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
611 if not re.search(
'_instFlux$', c):
612 cols.append(f
'{c}_instFlux')
617 return list(set([c
for c
in cols
if c
not in not_a_col]))
624 """Get column with specified name
640 return df[self.
colcol]
644 """Return the value of the index for each object
647 columns = [
'coord_ra']
648 _defaultDataset =
'ref'
652 return pd.Series(df.index, index=df.index)
657 _allow_difference =
False
661 return pd.Series(df.index, index=df.index)
665 col =
'base_Footprint_nPix'
669 """Base class for coordinate column, in degrees
678 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
683 """Right Ascension, in degrees
689 super().
__init__(
'coord_ra', **kwargs)
692 return super().
__call__(catalog, **kwargs)
696 """Declination, in degrees
702 super().
__init__(
'coord_dec', **kwargs)
705 return super().
__call__(catalog, **kwargs)
709 if not col.endswith(
'_instFlux'):
715 if not col.endswith(
'_instFluxErr'):
716 col +=
'_instFluxErr'
721 """Compute calibrated magnitude
723 Takes a `calib` argument, which returns the flux at mag=0
724 as `calib.getFluxMag0()`. If not provided, then the default
725 `fluxMag0` is 63095734448.0194, which is default for HSC.
726 This default should be removed in DM-21955
728 This calculation hides warnings about invalid values and dividing by zero.
730 As for all functors, a `dataset` and `filt` kwarg should be provided upon
731 initialization. Unlike the default `Functor`, however, the default dataset
732 for a `Mag` is `'meas'`, rather than `'ref'`.
737 Name of flux column from which to compute magnitude. Can be parseable
738 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
739 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
741 calib : `lsst.afw.image.calib.Calib` (optional)
742 Object that knows zero point.
744 _defaultDataset =
'meas'
749 if calib
is not None:
753 self.
fluxMag0fluxMag0 = 63095734448.0194
762 with np.warnings.catch_warnings():
763 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
764 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
765 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
769 return f
'mag_{self.col}'
773 """Compute calibrated magnitude uncertainty
775 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
780 calib : `lsst.afw.image.calib.Calib` (optional)
781 Object that knows zero point.
786 if self.
calibcalib
is not None:
793 return [self.
colcol, self.
colcol +
'Err']
796 with np.warnings.catch_warnings():
797 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
798 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
800 x = df[fluxErrCol] / df[fluxCol]
802 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
807 return super().name +
'_err'
815 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
819 _defaultDataset =
'meas'
821 """Functor to calculate magnitude difference"""
830 return [self.
col1col1, self.
col2col2]
833 with np.warnings.catch_warnings():
834 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
835 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
836 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
840 return f
'(mag_{self.col1} - mag_{self.col2})'
844 return f
'magDiff_{self.col1}_{self.col2}'
848 """Compute the color between two filters
850 Computes color by initializing two different `Mag`
851 functors based on the `col` and filters provided, and
852 then returning the difference.
854 This is enabled by the `_func` expecting a dataframe with a
855 multilevel column index, with both `'band'` and `'column'`,
856 instead of just `'column'`, which is the `Functor` default.
857 This is controlled by the `_dfLevels` attribute.
859 Also of note, the default dataset for `Color` is `forced_src'`,
860 whereas for `Mag` it is `'meas'`.
865 Name of flux column from which to compute; same as would be passed to
866 `lsst.pipe.tasks.functors.Mag`.
869 Filters from which to compute magnitude difference.
870 Color computed is `Mag(filt2) - Mag(filt1)`.
872 _defaultDataset =
'forced_src'
873 _dfLevels = (
'band',
'column')
879 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
883 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
884 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
897 mag2 = self.mag2._func(df[self.filt2])
898 mag1 = self.mag1._func(df[self.filt1])
903 return [self.
mag1mag1.col, self.
mag2mag2.col]
910 return f
'{self.filt2} - {self.filt1} ({self.col})'
914 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
918 """Main function of this subclass is to override the dropna=True
921 _allow_difference =
False
926 return super().
__call__(parq, dropna=
False, **kwargs)
930 _columns = [
"base_ClassificationExtendedness_value"]
931 _column =
"base_ClassificationExtendedness_value"
936 test = (x < 0.5).astype(int)
937 test = test.mask(mask, 2)
941 categories = [
'galaxy',
'star', self.
_null_label_null_label]
942 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
943 index=x.index, name=
'label')
945 label = label.astype(str)
950 _columns = [
'numStarFlags']
951 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
957 n = len(x.unique()) - 1
959 labels = [
'noStar',
'maybe',
'star']
960 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
961 index=x.index, name=
'label')
964 label = label.astype(str)
970 name =
'Deconvolved Moments'
971 shortname =
'deconvolvedMoments'
972 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
973 "ext_shapeHSM_HsmSourceMoments_yy",
974 "base_SdssShape_xx",
"base_SdssShape_yy",
975 "ext_shapeHSM_HsmPsfMoments_xx",
976 "ext_shapeHSM_HsmPsfMoments_yy")
979 """Calculate deconvolved moments"""
980 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
981 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
983 hsm = np.ones(len(df))*np.nan
984 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
985 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
986 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
991 raise RuntimeError(
'No psf shape parameter found in catalog')
993 return hsm.where(np.isfinite(hsm), sdss) - psf
997 """Functor to calculate SDSS trace radius size for sources"""
998 name =
"SDSS Trace Size"
999 shortname =
'sdssTrace'
1000 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1002 def _func(self, df):
1003 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1008 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1009 name =
"PSF - SDSS Trace Size"
1010 shortname =
'psf_sdssTrace'
1011 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1012 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1014 def _func(self, df):
1015 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1016 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1017 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1022 """Functor to calculate HSM trace radius size for sources"""
1023 name =
'HSM Trace Size'
1024 shortname =
'hsmTrace'
1025 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1026 "ext_shapeHSM_HsmSourceMoments_yy")
1028 def _func(self, df):
1029 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1030 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1035 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1036 name =
'PSF - HSM Trace Size'
1037 shortname =
'psf_HsmTrace'
1038 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1039 "ext_shapeHSM_HsmSourceMoments_yy",
1040 "ext_shapeHSM_HsmPsfMoments_xx",
1041 "ext_shapeHSM_HsmPsfMoments_yy")
1043 def _func(self, df):
1044 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1045 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1046 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1047 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1048 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1053 name =
'HSM Psf FWHM'
1054 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1057 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1059 def _func(self, df):
1061 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1065 name =
"Distortion Ellipticity (e1)"
1066 shortname =
"Distortion"
1079 def _func(self, df):
1084 name =
"Ellipticity e2"
1096 def _func(self, df):
1097 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1112 def _func(self, df):
1113 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1117 """Computations using the stored localWcs.
1119 name =
"LocalWcsOperations"
1134 """Compute the distance on the sphere from x2, y1 to x1, y1.
1142 cd11 : `pandas.Series`
1143 [1, 1] element of the local Wcs affine transform.
1144 cd11 : `pandas.Series`
1145 [1, 1] element of the local Wcs affine transform.
1146 cd12 : `pandas.Series`
1147 [1, 2] element of the local Wcs affine transform.
1148 cd21 : `pandas.Series`
1149 [2, 1] element of the local Wcs affine transform.
1150 cd22 : `pandas.Series`
1151 [2, 2] element of the local Wcs affine transform.
1156 RA and dec conversion of x and y given the local Wcs. Returned
1157 units are in radians.
1160 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1163 """Compute the local pixel scale conversion.
1167 ra1 : `pandas.Series`
1168 Ra of the first coordinate in radians.
1169 dec1 : `pandas.Series`
1170 Dec of the first coordinate in radians.
1171 ra2 : `pandas.Series`
1172 Ra of the second coordinate in radians.
1173 dec2 : `pandas.Series`
1174 Dec of the second coordinate in radians.
1178 dist : `pandas.Series`
1179 Distance on the sphere in radians.
1181 deltaDec = dec2 - dec1
1183 return 2 * np.arcsin(
1185 np.sin(deltaDec / 2) ** 2
1186 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1189 """Compute the distance on the sphere from x2, y1 to x1, y1.
1193 x1 : `pandas.Series`
1195 y1 : `pandas.Series`
1197 x2 : `pandas.Series`
1199 y2 : `pandas.Series`
1201 cd11 : `pandas.Series`
1202 [1, 1] element of the local Wcs affine transform.
1203 cd11 : `pandas.Series`
1204 [1, 1] element of the local Wcs affine transform.
1205 cd12 : `pandas.Series`
1206 [1, 2] element of the local Wcs affine transform.
1207 cd21 : `pandas.Series`
1208 [2, 1] element of the local Wcs affine transform.
1209 cd22 : `pandas.Series`
1210 [2, 2] element of the local Wcs affine transform.
1214 Distance : `pandas.Series`
1215 Arcseconds per pixel at the location of the local WC
1217 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1218 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1224 """Compute the local pixel scale from the stored CDMatrix.
1236 """Compute the local pixel to scale conversion in arcseconds.
1240 cd11 : `pandas.Series`
1241 [1, 1] element of the local Wcs affine transform in radians.
1242 cd11 : `pandas.Series`
1243 [1, 1] element of the local Wcs affine transform in radians.
1244 cd12 : `pandas.Series`
1245 [1, 2] element of the local Wcs affine transform in radians.
1246 cd21 : `pandas.Series`
1247 [2, 1] element of the local Wcs affine transform in radians.
1248 cd22 : `pandas.Series`
1249 [2, 2] element of the local Wcs affine transform in radians.
1253 pixScale : `pandas.Series`
1254 Arcseconds per pixel at the location of the local WC
1256 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1258 def _func(self, df):
1266 """Convert a value in units pixels squared to units arcseconds squared.
1285 return f
"{self.col}_asArcseconds"
1289 return [self.
colcol,
1295 def _func(self, df):
1303 """Convert a value in units pixels to units arcseconds.
1322 return f
"{self.col}_asArcsecondsSq"
1326 return [self.
colcol,
1332 def _func(self, df):
1337 return df[self.
colcol] * pixScale * pixScale
1341 name =
'Reference Band'
1342 shortname =
'refBand'
1346 return [
"merge_measurement_i",
1347 "merge_measurement_r",
1348 "merge_measurement_z",
1349 "merge_measurement_y",
1350 "merge_measurement_g"]
1352 def _func(self, df):
1353 def getFilterAliasName(row):
1355 colName = row.idxmax()
1356 return colName.replace(
'merge_measurement_',
'')
1358 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1363 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1364 LOG_AB_FLUX_SCALE = 12.56
1365 FIVE_OVER_2LOG10 = 1.085736204758129569
1369 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1375 if calib
is not None:
1385 return [self.
colcol]
1389 return f
'mag_{self.col}'
1393 if np.abs(a) < np.abs(b):
1398 return np.abs(a) * np.sqrt(1. + q*q)
1404 with np.warnings.catch_warnings():
1405 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1406 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1407 return -2.5 * np.log10(dn/fluxMag0)
1410 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1411 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1415 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1420 def _func(self, df):
1429 def _func(self, df):
1431 return pd.Series(retArr, index=df.index)
1435 def _func(self, df):
1444 def _func(self, df):
1446 return pd.Series(retArr, index=df.index)
1450 """Base class for calibrating the specified instrument flux column using
1451 the local photometric calibration.
1456 Name of the instrument flux column.
1457 instFluxErrCol : `str`
1458 Name of the assocated error columns for ``instFluxCol``.
1459 photoCalibCol : `str`
1460 Name of local calibration column.
1461 photoCalibErrCol : `str`
1462 Error associated with ``photoCalibCol``
1472 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1487 """Convert instrument flux to nanojanskys.
1491 instFlux : `numpy.ndarray` or `pandas.Series`
1492 Array of instrument flux measurements
1493 localCalib : `numpy.ndarray` or `pandas.Series`
1494 Array of local photometric calibration estimates.
1498 calibFlux : `numpy.ndarray` or `pandas.Series`
1499 Array of calibrated flux measurements.
1501 return instFlux * localCalib
1504 """Convert instrument flux to nanojanskys.
1508 instFlux : `numpy.ndarray` or `pandas.Series`
1509 Array of instrument flux measurements
1510 instFluxErr : `numpy.ndarray` or `pandas.Series`
1511 Errors on associated ``instFlux`` values
1512 localCalib : `numpy.ndarray` or `pandas.Series`
1513 Array of local photometric calibration estimates.
1514 localCalibErr : `numpy.ndarray` or `pandas.Series`
1515 Errors on associated ``localCalib`` values
1519 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1520 Errors on calibrated flux measurements.
1522 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1525 """Convert instrument flux to nanojanskys.
1529 instFlux : `numpy.ndarray` or `pandas.Series`
1530 Array of instrument flux measurements
1531 localCalib : `numpy.ndarray` or `pandas.Series`
1532 Array of local photometric calibration estimates.
1536 calibMag : `numpy.ndarray` or `pandas.Series`
1537 Array of calibrated AB magnitudes.
1542 """Convert instrument flux err to nanojanskys.
1546 instFlux : `numpy.ndarray` or `pandas.Series`
1547 Array of instrument flux measurements
1548 instFluxErr : `numpy.ndarray` or `pandas.Series`
1549 Errors on associated ``instFlux`` values
1550 localCalib : `numpy.ndarray` or `pandas.Series`
1551 Array of local photometric calibration estimates.
1552 localCalibErr : `numpy.ndarray` or `pandas.Series`
1553 Errors on associated ``localCalib`` values
1557 calibMagErr: `numpy.ndarray` or `pandas.Series`
1558 Error on calibrated AB magnitudes.
1561 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1565 """Compute calibrated fluxes using the local calibration value.
1581 return f
'flux_{self.instFluxCol}'
1583 def _func(self, df):
1588 """Compute calibrated flux errors using the local calibration value.
1605 return f
'fluxErr_{self.instFluxCol}'
1607 def _func(self, df):
1613 """Compute calibrated AB magnitudes using the local calibration value.
1629 return f
'mag_{self.instFluxCol}'
1631 def _func(self, df):
1637 """Compute calibrated AB magnitude errors using the local calibration value.
1654 return f
'magErr_{self.instFluxCol}'
1656 def _func(self, df):
1664 """Compute absolute mean of dipole fluxes.
1673 LocalDipoleMeanFluxErr
1675 LocalDipoleDiffFluxErr
1705 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1707 def _func(self, df):
1713 """Compute the error on the absolute mean of dipole fluxes.
1722 LocalDipoleMeanFluxErr
1724 LocalDipoleDiffFluxErr
1738 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1740 def _func(self, df):
1749 """Compute the absolute difference of dipole fluxes.
1751 Value is (abs(pos) - abs(neg))
1760 LocalDipoleMeanFluxErr
1762 LocalDipoleDiffFluxErr
1773 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1775 def _func(self, df):
1781 """Compute the error on the absolute difference of dipole fluxes.
1790 LocalDipoleMeanFluxErr
1792 LocalDipoleDiffFluxErr
1806 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1808 def _func(self, df):
1817 """Base class for returning the ratio of 2 columns.
1819 Can be used to compute a Signal to Noise ratio for any input flux.
1824 Name of the column to use at the numerator in the ratio
1826 Name of the column to use as the denominator in the ratio.
1842 return f
'ratio_{self.numerator}_{self.denominator}'
1844 def _func(self, df):
1845 with np.warnings.catch_warnings():
1846 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1847 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)