3 from itertools
import product
7 import astropy.units
as u
10 from lsst.daf.butler
import DeferredDatasetHandle
11 from .parquetTable
import ParquetTable, MultilevelParquetTable
15 typeKey='functor', name=None):
16 """Initialize an object defined in a dictionary
18 The object needs to be importable as
19 f'{basePath}.{initDict[typeKey]}'
20 The positional and keyword arguments (if any) are contained in
21 "args" and "kwargs" entries in the dictionary, respectively.
22 This is used in `functors.CompositeFunctor.from_yaml` to initialize
23 a composite functor from a specification in a YAML file.
28 Dictionary describing object's initialization. Must contain
29 an entry keyed by ``typeKey`` that is the name of the object,
30 relative to ``basePath``.
32 Path relative to module in which ``initDict[typeKey]`` is defined.
34 Key of ``initDict`` that is the name of the object
35 (relative to `basePath`).
37 initDict = initDict.copy()
39 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
41 if 'args' in initDict:
42 args = initDict.pop(
'args')
43 if isinstance(args, str):
46 element = pythonType(*args, **initDict)
47 except Exception
as e:
48 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
49 raise type(e)(message, e.args)
54 """Define and execute a calculation on a ParquetTable
56 The `__call__` method accepts either a `ParquetTable` object or a
57 `DeferredDatasetHandle`, and returns the
58 result of the calculation as a single column. Each functor defines what
59 columns are needed for the calculation, and only these columns are read
60 from the `ParquetTable`.
62 The action of `__call__` consists of two steps: first, loading the
63 necessary columns from disk into memory as a `pandas.DataFrame` object;
64 and second, performing the computation on this dataframe and returning the
68 To define a new `Functor`, a subclass must define a `_func` method,
69 that takes a `pandas.DataFrame` and returns result in a `pandas.Series`.
70 In addition, it must define the following attributes
72 * `_columns`: The columns necessary to perform the calculation
73 * `name`: A name appropriate for a figure axis label
74 * `shortname`: A name appropriate for use as a dictionary key
76 On initialization, a `Functor` should declare what band (`filt` kwarg)
77 and dataset (e.g. `'ref'`, `'meas'`, `'forced_src'`) it is intended to be
78 applied to. This enables the `_get_data` method to extract the proper
79 columns from the parquet file. If not specified, the dataset will fall back
80 on the `_defaultDataset`attribute. If band is not specified and `dataset`
81 is anything other than `'ref'`, then an error will be raised when trying to
82 perform the calculation.
84 As currently implemented, `Functor` is only set up to expect a
85 dataset of the format of the `deepCoadd_obj` dataset; that is, a
86 dataframe with a multi-level column index,
87 with the levels of the column index being `band`,
88 `dataset`, and `column`. This is defined in the `_columnLevels` attribute,
89 as well as being implicit in the role of the `filt` and `dataset` attributes
90 defined at initialization. In addition, the `_get_data` method that reads
91 the dataframe from the `ParquetTable` will return a dataframe with column
92 index levels defined by the `_dfLevels` attribute; by default, this is
95 The `_columnLevels` and `_dfLevels` attributes should generally not need to
96 be changed, unless `_func` needs columns from multiple filters or datasets
97 to do the calculation.
98 An example of this is the `lsst.pipe.tasks.functors.Color` functor, for
99 which `_dfLevels = ('band', 'column')`, and `_func` expects the dataframe
100 it gets to have those levels in the column index.
105 Filter upon which to do the calculation
108 Dataset upon which to do the calculation
109 (e.g., 'ref', 'meas', 'forced_src').
113 _defaultDataset =
'ref'
114 _columnLevels = (
'band',
'dataset',
'column')
115 _dfLevels = (
'column',)
116 _defaultNoDup =
False
118 def __init__(self, filt=None, dataset=None, noDup=None):
125 if self.
_noDup_noDup
is not None:
132 """Columns required to perform calculation
134 if not hasattr(self,
'_columns'):
135 raise NotImplementedError(
'Must define columns property or _columns attribute')
138 def _get_data_columnLevels(self, data, columnIndex=None):
139 """Gets the names of the column index levels
141 This should only be called in the context of a multilevel table.
142 The logic here is to enable this to work both with the gen2 `MultilevelParquetTable`
143 and with the gen3 `DeferredDatasetHandle`.
147 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
149 columnnIndex (optional): pandas `Index` object
150 if not passed, then it is read from the `DeferredDatasetHandle`
152 if isinstance(data, DeferredDatasetHandle):
153 if columnIndex
is None:
154 columnIndex = data.get(component=
"columns")
155 if columnIndex
is not None:
156 return columnIndex.names
157 if isinstance(data, MultilevelParquetTable):
158 return data.columnLevels
160 raise TypeError(f
"Unknown type for data: {type(data)}!")
162 def _get_data_columnLevelNames(self, data, columnIndex=None):
163 """Gets the content of each of the column levels for a multilevel table
165 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
167 Mirrors original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
169 if isinstance(data, DeferredDatasetHandle):
170 if columnIndex
is None:
171 columnIndex = data.get(component=
"columns")
172 if columnIndex
is not None:
173 columnLevels = columnIndex.names
175 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
176 for i, level
in enumerate(columnLevels)
178 return columnLevelNames
179 if isinstance(data, MultilevelParquetTable):
180 return data.columnLevelNames
182 raise TypeError(f
"Unknown type for data: {type(data)}!")
184 def _colsFromDict(self, colDict, columnIndex=None):
185 """Converts dictionary column specficiation to a list of columns
187 This mirrors the original gen2 implementation within `pipe.tasks.parquetTable.MultilevelParquetTable`
192 for i, lev
in enumerate(columnLevels):
194 if isinstance(colDict[lev], str):
195 new_colDict[lev] = [colDict[lev]]
197 new_colDict[lev] = colDict[lev]
199 new_colDict[lev] = columnIndex.levels[i]
201 levelCols = [new_colDict[lev]
for lev
in columnLevels]
202 cols = product(*levelCols)
206 """Returns columns needed by functor from multilevel dataset
208 To access tables with multilevel column structure, the `MultilevelParquetTable`
209 or `DeferredDatasetHandle` need to be passed either a list of tuples or a
214 data : `MultilevelParquetTable` or `DeferredDatasetHandle`
216 columnIndex (optional): pandas `Index` object
217 either passed or read in from `DeferredDatasetHandle`.
220 If true, then return a list of tuples rather than the column dictionary
221 specification. This is set to `True` by `CompositeFunctor` in order to be able to
222 combine columns from the various component functors.
225 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
226 columnIndex = data.get(component=
"columns")
231 if not set(columnLevels) == set(self.
_columnLevels_columnLevels):
233 "ParquetTable does not have the expected column levels. "
234 f
"Got {columnLevels}; expected {self._columnLevels}."
237 columnDict = {
'column': self.
columnscolumns,
238 'dataset': self.
datasetdataset}
239 if self.
filtfilt
is None:
241 if "band" in columnLevels:
242 if self.
datasetdataset ==
"ref":
243 columnDict[
"band"] = columnLevelNames[
"band"][0]
245 raise ValueError(f
"'filt' not set for functor {self.name}"
246 f
"(dataset {self.dataset}) "
248 "contains multiple filters in column index. "
249 "Set 'filt' or set 'dataset' to 'ref'.")
251 columnDict[
'band'] = self.
filtfilt
253 if isinstance(data, MultilevelParquetTable):
254 return data._colsFromDict(columnDict)
255 elif isinstance(data, DeferredDatasetHandle):
257 return self.
_colsFromDict_colsFromDict(columnDict, columnIndex=columnIndex)
261 def _func(self, df, dropna=True):
262 raise NotImplementedError(
'Must define calculation on dataframe')
264 def _get_columnIndex(self, data):
265 """Return columnIndex
268 if isinstance(data, DeferredDatasetHandle):
269 return data.get(component=
"columns")
273 def _get_data(self, data):
274 """Retrieve dataframe necessary for calculation.
276 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
278 Returns dataframe upon which `self._func` can act.
280 N.B. while passing a raw pandas `DataFrame` *should* work here, it has not been tested.
282 if isinstance(data, pd.DataFrame):
287 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
290 if isinstance(data, ParquetTable)
and not is_multiLevel:
292 df = data.toDataFrame(columns=columns)
299 if isinstance(data, MultilevelParquetTable):
301 df = data.toDataFrame(columns=columns, droplevels=
False)
302 elif isinstance(data, DeferredDatasetHandle):
304 df = data.get(parameters={
"columns": columns})
310 def _setLevels(self, df):
311 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels_dfLevels]
312 df.columns = df.columns.droplevel(levelsToDrop)
315 def _dropna(self, vals):
321 vals = self.
_func_func(df)
323 vals = self.
failfail(df)
325 vals = self.
_dropna_dropna(vals)
330 """Computes difference between functor called on two different ParquetTable objects
332 return self(data1, **kwargs) - self(data2, **kwargs)
335 return pd.Series(np.full(len(df), np.nan), index=df.index)
339 """Full name of functor (suitable for figure labels)
341 return NotImplementedError
345 """Short name of functor (suitable for column name/dict key)
351 """Perform multiple calculations at once on a catalog
353 The role of a `CompositeFunctor` is to group together computations from
354 multiple functors. Instead of returning `pandas.Series` a
355 `CompositeFunctor` returns a `pandas.Dataframe`, with the column names
356 being the keys of `funcDict`.
358 The `columns` attribute of a `CompositeFunctor` is the union of all columns
359 in all the component functors.
361 A `CompositeFunctor` does not use a `_func` method itself; rather,
362 when a `CompositeFunctor` is called, all its columns are loaded
363 at once, and the resulting dataframe is passed to the `_func` method of each component
364 functor. This has the advantage of only doing I/O (reading from parquet file) once,
365 and works because each individual `_func` method of each component functor does not
366 care if there are *extra* columns in the dataframe being passed; only that it must contain
367 *at least* the `columns` it expects.
369 An important and useful class method is `from_yaml`, which takes as argument the path to a YAML
370 file specifying a collection of functors.
374 funcs : `dict` or `list`
375 Dictionary or list of functors. If a list, then it will be converted
376 into a dictonary according to the `.shortname` attribute of each functor.
383 if type(funcs) == dict:
386 self.
funcDictfuncDict = {f.shortname: f
for f
in funcs}
388 self.
_filt_filt =
None
394 return self.
_filt_filt
399 for _, f
in self.
funcDictfuncDict.items():
401 self.
_filt_filt = filt
404 if isinstance(new, dict):
406 elif isinstance(new, CompositeFunctor):
409 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
417 return list(set([x
for y
in [f.columns
for f
in self.
funcDictfuncDict.values()]
for x
in y]))
426 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDictfuncDict.values()
437 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
440 if isinstance(data, ParquetTable)
and not is_multiLevel:
442 df = data.toDataFrame(columns=columns)
443 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
449 if isinstance(data, MultilevelParquetTable):
451 df = data.toDataFrame(columns=columns, droplevels=
False)
452 elif isinstance(data, DeferredDatasetHandle):
454 df = data.get(parameters={
"columns": columns})
457 for k, f
in self.
funcDictfuncDict.items():
459 subdf = f._setLevels(
460 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
462 valDict[k] = f._func(subdf)
464 valDict[k] = f.fail(subdf)
467 elif isinstance(data, DeferredDatasetHandle):
469 df = data.get(parameters={
"columns": columns})
470 valDict = {k: f._func(df)
for k, f
in self.
funcDictfuncDict.items()}
473 valDf = pd.concat(valDict, axis=1)
475 print([(k, type(v))
for k, v
in valDict.items()])
478 if kwargs.get(
'dropna',
False):
479 valDf = valDf.dropna(how=
'any')
485 if renameRules
is None:
487 for old, new
in renameRules:
488 if col.startswith(old):
489 col = col.replace(old, new)
494 with open(filename)
as f:
495 translationDefinition = yaml.safe_load(f)
497 return cls.
from_yamlfrom_yaml(translationDefinition, **kwargs)
502 for func, val
in translationDefinition[
'funcs'].items():
505 if 'flag_rename_rules' in translationDefinition:
506 renameRules = translationDefinition[
'flag_rename_rules']
510 if 'refFlags' in translationDefinition:
511 for flag
in translationDefinition[
'refFlags']:
512 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'ref')
514 if 'flags' in translationDefinition:
515 for flag
in translationDefinition[
'flags']:
516 funcs[cls.
renameColrenameCol(flag, renameRules)] =
Column(flag, dataset=
'meas')
518 return cls(funcs, **kwargs)
522 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
524 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
528 df : pandas.DataFrame
529 Dataframe on which to evaluate expression.
535 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
536 val = df.eval(expr_new, truediv=
True)
538 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
539 val = df.eval(expr_new, truediv=
True)
544 """Arbitrary computation on a catalog
546 Column names (and thus the columns to be loaded from catalog) are found
547 by finding all words and trying to ignore all "math-y" words.
552 Expression to evaluate, to be parsed and executed by `mag_aware_eval`.
554 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
566 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
exprexpr)
568 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
exprexpr)
if c
not in self.
_ignore_words_ignore_words]
571 if not re.search(
'_instFlux$', c):
572 cols.append(f
'{c}_instFlux')
577 return list(set([c
for c
in cols
if c
not in not_a_col]))
584 """Get column with specified name
600 return df[self.
colcol]
604 """Return the value of the index for each object
607 columns = [
'coord_ra']
608 _defaultDataset =
'ref'
612 return pd.Series(df.index, index=df.index)
617 _allow_difference =
False
621 return pd.Series(df.index, index=df.index)
625 col =
'base_Footprint_nPix'
629 """Base class for coordinate column, in degrees
638 output = df[self.
colcol] * 180 / np.pi
if self.
_radians_radians
else df[self.
colcol]
643 """Right Ascension, in degrees
649 super().
__init__(
'coord_ra', **kwargs)
652 return super().
__call__(catalog, **kwargs)
656 """Declination, in degrees
662 super().
__init__(
'coord_dec', **kwargs)
665 return super().
__call__(catalog, **kwargs)
669 if not col.endswith(
'_instFlux'):
675 if not col.endswith(
'_instFluxErr'):
676 col +=
'_instFluxErr'
681 """Compute calibrated magnitude
683 Takes a `calib` argument, which returns the flux at mag=0
684 as `calib.getFluxMag0()`. If not provided, then the default
685 `fluxMag0` is 63095734448.0194, which is default for HSC.
686 This default should be removed in DM-21955
688 This calculation hides warnings about invalid values and dividing by zero.
690 As for all functors, a `dataset` and `filt` kwarg should be provided upon
691 initialization. Unlike the default `Functor`, however, the default dataset
692 for a `Mag` is `'meas'`, rather than `'ref'`.
697 Name of flux column from which to compute magnitude. Can be parseable
698 by `lsst.pipe.tasks.functors.fluxName` function---that is, you can pass
699 `'modelfit_CModel'` instead of `'modelfit_CModel_instFlux'`) and it will
701 calib : `lsst.afw.image.calib.Calib` (optional)
702 Object that knows zero point.
704 _defaultDataset =
'meas'
709 if calib
is not None:
713 self.
fluxMag0fluxMag0 = 63095734448.0194
722 with np.warnings.catch_warnings():
723 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
724 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
725 return -2.5*np.log10(df[self.
colcol] / self.
fluxMag0fluxMag0)
729 return f
'mag_{self.col}'
733 """Compute calibrated magnitude uncertainty
735 Takes the same `calib` object as `lsst.pipe.tasks.functors.Mag`.
740 calib : `lsst.afw.image.calib.Calib` (optional)
741 Object that knows zero point.
746 if self.
calibcalib
is not None:
753 return [self.
colcol, self.
colcol +
'Err']
756 with np.warnings.catch_warnings():
757 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
758 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
760 x = df[fluxErrCol] / df[fluxCol]
762 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
767 return super().name +
'_err'
775 return (df[self.
colcol] / self.
fluxMag0fluxMag0) * 1e9
779 _defaultDataset =
'meas'
781 """Functor to calculate magnitude difference"""
790 return [self.
col1col1, self.
col2col2]
793 with np.warnings.catch_warnings():
794 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
795 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
796 return -2.5*np.log10(df[self.
col1col1]/df[self.
col2col2])
800 return f
'(mag_{self.col1} - mag_{self.col2})'
804 return f
'magDiff_{self.col1}_{self.col2}'
808 """Compute the color between two filters
810 Computes color by initializing two different `Mag`
811 functors based on the `col` and filters provided, and
812 then returning the difference.
814 This is enabled by the `_func` expecting a dataframe with a
815 multilevel column index, with both `'band'` and `'column'`,
816 instead of just `'column'`, which is the `Functor` default.
817 This is controlled by the `_dfLevels` attribute.
819 Also of note, the default dataset for `Color` is `forced_src'`,
820 whereas for `Mag` it is `'meas'`.
825 Name of flux column from which to compute; same as would be passed to
826 `lsst.pipe.tasks.functors.Mag`.
829 Filters from which to compute magnitude difference.
830 Color computed is `Mag(filt2) - Mag(filt1)`.
832 _defaultDataset =
'forced_src'
833 _dfLevels = (
'band',
'column')
839 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
843 self.
mag2mag2 =
Mag(col, filt=filt2, **kwargs)
844 self.
mag1mag1 =
Mag(col, filt=filt1, **kwargs)
857 mag2 = self.mag2._func(df[self.filt2])
858 mag1 = self.mag1._func(df[self.filt1])
863 return [self.
mag1mag1.col, self.
mag2mag2.col]
870 return f
'{self.filt2} - {self.filt1} ({self.col})'
874 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
878 """Main function of this subclass is to override the dropna=True
881 _allow_difference =
False
886 return super().
__call__(parq, dropna=
False, **kwargs)
890 _columns = [
"base_ClassificationExtendedness_value"]
891 _column =
"base_ClassificationExtendedness_value"
896 test = (x < 0.5).astype(int)
897 test = test.mask(mask, 2)
901 categories = [
'galaxy',
'star', self.
_null_label_null_label]
902 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
903 index=x.index, name=
'label')
905 label = label.astype(str)
910 _columns = [
'numStarFlags']
911 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
917 n = len(x.unique()) - 1
919 labels = [
'noStar',
'maybe',
'star']
920 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
921 index=x.index, name=
'label')
924 label = label.astype(str)
930 name =
'Deconvolved Moments'
931 shortname =
'deconvolvedMoments'
932 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
933 "ext_shapeHSM_HsmSourceMoments_yy",
934 "base_SdssShape_xx",
"base_SdssShape_yy",
935 "ext_shapeHSM_HsmPsfMoments_xx",
936 "ext_shapeHSM_HsmPsfMoments_yy")
939 """Calculate deconvolved moments"""
940 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
941 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
943 hsm = np.ones(len(df))*np.nan
944 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
945 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
946 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
951 raise RuntimeError(
'No psf shape parameter found in catalog')
953 return hsm.where(np.isfinite(hsm), sdss) - psf
957 """Functor to calculate SDSS trace radius size for sources"""
958 name =
"SDSS Trace Size"
959 shortname =
'sdssTrace'
960 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
963 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
968 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
969 name =
"PSF - SDSS Trace Size"
970 shortname =
'psf_sdssTrace'
971 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
972 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
975 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
976 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
977 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
982 """Functor to calculate HSM trace radius size for sources"""
983 name =
'HSM Trace Size'
984 shortname =
'hsmTrace'
985 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
986 "ext_shapeHSM_HsmSourceMoments_yy")
989 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
990 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
995 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
996 name =
'PSF - HSM Trace Size'
997 shortname =
'psf_HsmTrace'
998 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
999 "ext_shapeHSM_HsmSourceMoments_yy",
1000 "ext_shapeHSM_HsmPsfMoments_xx",
1001 "ext_shapeHSM_HsmPsfMoments_yy")
1003 def _func(self, df):
1004 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1005 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1006 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1007 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1008 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1013 name =
'HSM Psf FWHM'
1014 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1017 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1019 def _func(self, df):
1021 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1025 name =
"Distortion Ellipticity (e1)"
1026 shortname =
"Distortion"
1039 def _func(self, df):
1044 name =
"Ellipticity e2"
1056 def _func(self, df):
1057 return 2*df[self.
colXYcolXY] / (df[self.
colXXcolXX] + df[self.
colYYcolYY])
1072 def _func(self, df):
1073 return (df[self.
colXXcolXX]*df[self.
colYYcolYY] - df[self.
colXYcolXY]**2)**0.25
1077 """Computations using the stored localWcs.
1079 name =
"LocalWcsOperations"
1094 """Compute the distance on the sphere from x2, y1 to x1, y1.
1102 cd11 : `pandas.Series`
1103 [1, 1] element of the local Wcs affine transform.
1104 cd11 : `pandas.Series`
1105 [1, 1] element of the local Wcs affine transform.
1106 cd12 : `pandas.Series`
1107 [1, 2] element of the local Wcs affine transform.
1108 cd21 : `pandas.Series`
1109 [2, 1] element of the local Wcs affine transform.
1110 cd22 : `pandas.Series`
1111 [2, 2] element of the local Wcs affine transform.
1116 RA and dec conversion of x and y given the local Wcs. Returned
1117 units are in radians.
1120 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1123 """Compute the local pixel scale conversion.
1127 ra1 : `pandas.Series`
1128 Ra of the first coordinate in radians.
1129 dec1 : `pandas.Series`
1130 Dec of the first coordinate in radians.
1131 ra2 : `pandas.Series`
1132 Ra of the second coordinate in radians.
1133 dec2 : `pandas.Series`
1134 Dec of the second coordinate in radians.
1138 dist : `pandas.Series`
1139 Distance on the sphere in radians.
1141 deltaDec = dec2 - dec1
1143 return 2 * np.arcsin(
1145 np.sin(deltaDec / 2) ** 2
1146 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1149 """Compute the distance on the sphere from x2, y1 to x1, y1.
1153 x1 : `pandas.Series`
1155 y1 : `pandas.Series`
1157 x2 : `pandas.Series`
1159 y2 : `pandas.Series`
1161 cd11 : `pandas.Series`
1162 [1, 1] element of the local Wcs affine transform.
1163 cd11 : `pandas.Series`
1164 [1, 1] element of the local Wcs affine transform.
1165 cd12 : `pandas.Series`
1166 [1, 2] element of the local Wcs affine transform.
1167 cd21 : `pandas.Series`
1168 [2, 1] element of the local Wcs affine transform.
1169 cd22 : `pandas.Series`
1170 [2, 2] element of the local Wcs affine transform.
1174 Distance : `pandas.Series`
1175 Arcseconds per pixel at the location of the local WC
1177 ra1, dec1 = self.
computeDeltaRaDeccomputeDeltaRaDec(x1, y1, cd11, cd12, cd21, cd22)
1178 ra2, dec2 = self.
computeDeltaRaDeccomputeDeltaRaDec(x2, y2, cd11, cd12, cd21, cd22)
1184 """Compute the local pixel scale from the stored CDMatrix.
1196 """Compute the local pixel to scale conversion in arcseconds.
1200 cd11 : `pandas.Series`
1201 [1, 1] element of the local Wcs affine transform in radians.
1202 cd11 : `pandas.Series`
1203 [1, 1] element of the local Wcs affine transform in radians.
1204 cd12 : `pandas.Series`
1205 [1, 2] element of the local Wcs affine transform in radians.
1206 cd21 : `pandas.Series`
1207 [2, 1] element of the local Wcs affine transform in radians.
1208 cd22 : `pandas.Series`
1209 [2, 2] element of the local Wcs affine transform in radians.
1213 pixScale : `pandas.Series`
1214 Arcseconds per pixel at the location of the local WC
1216 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1218 def _func(self, df):
1226 """Convert a value in units pixels to units arcseconds.
1245 return f
"{self.col}_asArcseconds"
1249 return [self.
colcol,
1255 def _func(self, df):
1263 name =
'Reference Band'
1264 shortname =
'refBand'
1268 return [
"merge_measurement_i",
1269 "merge_measurement_r",
1270 "merge_measurement_z",
1271 "merge_measurement_y",
1272 "merge_measurement_g"]
1274 def _func(self, df):
1275 def getFilterAliasName(row):
1277 colName = row.idxmax()
1278 return colName.replace(
'merge_measurement_',
'')
1280 return df[self.
columnscolumnscolumns].apply(getFilterAliasName, axis=1)
1285 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1286 LOG_AB_FLUX_SCALE = 12.56
1287 FIVE_OVER_2LOG10 = 1.085736204758129569
1291 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1297 if calib
is not None:
1307 return [self.
colcol]
1311 return f
'mag_{self.col}'
1315 if np.abs(a) < np.abs(b):
1320 return np.abs(a) * np.sqrt(1. + q*q)
1326 with np.warnings.catch_warnings():
1327 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1328 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1329 return -2.5 * np.log10(dn/fluxMag0)
1332 retVal = self.
vhypotvhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1333 retVal *= self.
AB_FLUX_SCALEAB_FLUX_SCALE / fluxMag0 / fluxMag0
1337 retVal = self.
dn2fluxErrdn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2fluxdn2flux(dn, fluxMag0)
1342 def _func(self, df):
1351 def _func(self, df):
1353 return pd.Series(retArr, index=df.index)
1357 def _func(self, df):
1366 def _func(self, df):
1368 return pd.Series(retArr, index=df.index)
1372 """Base class for calibrating the specified instrument flux column using
1373 the local photometric calibration.
1378 Name of the instrument flux column.
1379 instFluxErrCol : `str`
1380 Name of the assocated error columns for ``instFluxCol``.
1381 photoCalibCol : `str`
1382 Name of local calibration column.
1383 photoCalibErrCol : `str`
1384 Error associated with ``photoCalibCol``
1394 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1409 """Convert instrument flux to nanojanskys.
1413 instFlux : `numpy.ndarray` or `pandas.Series`
1414 Array of instrument flux measurements
1415 localCalib : `numpy.ndarray` or `pandas.Series`
1416 Array of local photometric calibration estimates.
1420 calibFlux : `numpy.ndarray` or `pandas.Series`
1421 Array of calibrated flux measurements.
1423 return instFlux * localCalib
1426 """Convert instrument flux to nanojanskys.
1430 instFlux : `numpy.ndarray` or `pandas.Series`
1431 Array of instrument flux measurements
1432 instFluxErr : `numpy.ndarray` or `pandas.Series`
1433 Errors on associated ``instFlux`` values
1434 localCalib : `numpy.ndarray` or `pandas.Series`
1435 Array of local photometric calibration estimates.
1436 localCalibErr : `numpy.ndarray` or `pandas.Series`
1437 Errors on associated ``localCalib`` values
1441 calibFluxErr : `numpy.ndarray` or `pandas.Series`
1442 Errors on calibrated flux measurements.
1444 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1447 """Convert instrument flux to nanojanskys.
1451 instFlux : `numpy.ndarray` or `pandas.Series`
1452 Array of instrument flux measurements
1453 localCalib : `numpy.ndarray` or `pandas.Series`
1454 Array of local photometric calibration estimates.
1458 calibMag : `numpy.ndarray` or `pandas.Series`
1459 Array of calibrated AB magnitudes.
1464 """Convert instrument flux err to nanojanskys.
1468 instFlux : `numpy.ndarray` or `pandas.Series`
1469 Array of instrument flux measurements
1470 instFluxErr : `numpy.ndarray` or `pandas.Series`
1471 Errors on associated ``instFlux`` values
1472 localCalib : `numpy.ndarray` or `pandas.Series`
1473 Array of local photometric calibration estimates.
1474 localCalibErr : `numpy.ndarray` or `pandas.Series`
1475 Errors on associated ``localCalib`` values
1479 calibMagErr: `numpy.ndarray` or `pandas.Series`
1480 Error on calibrated AB magnitudes.
1483 return 2.5 / np.log(10) * err / self.
instFluxToNanojanskyinstFluxToNanojansky(instFlux, instFluxErr)
1487 """Compute calibrated fluxes using the local calibration value.
1503 return f
'flux_{self.instFluxCol}'
1505 def _func(self, df):
1510 """Compute calibrated flux errors using the local calibration value.
1527 return f
'fluxErr_{self.instFluxCol}'
1529 def _func(self, df):
1535 """Compute calibrated AB magnitudes using the local calibration value.
1551 return f
'mag_{self.instFluxCol}'
1553 def _func(self, df):
1559 """Compute calibrated AB magnitude errors using the local calibration value.
1576 return f
'magErr_{self.instFluxCol}'
1578 def _func(self, df):
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __call__(self, parq, dropna=False, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def mag_aware_eval(df, expr)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)