24from itertools
import product
30import astropy.units
as u
31from dustmaps.sfd
import SFDQuery
32from astropy.coordinates
import SkyCoord
35from lsst.daf.butler
import DeferredDatasetHandle
39from .parquetTable
import ParquetTable, MultilevelParquetTable
43 typeKey='functor', name=None):
44 """Initialize an object defined in a dictionary
46 The object needs to be importable as
47 f
'{basePath}.{initDict[typeKey]}'
48 The positional
and keyword arguments (
if any) are contained
in
49 "args" and "kwargs" entries
in the dictionary, respectively.
50 This
is used
in `functors.CompositeFunctor.from_yaml` to initialize
51 a composite functor
from a specification
in a YAML file.
56 Dictionary describing object
's initialization. Must contain
57 an entry keyed by ``typeKey`` that is the name of the object,
58 relative to ``basePath``.
60 Path relative to module
in which ``initDict[typeKey]``
is defined.
62 Key of ``initDict`` that
is the name of the object
63 (relative to `basePath`).
65 initDict = initDict.copy()
67 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
69 if 'args' in initDict:
70 args = initDict.pop(
'args')
71 if isinstance(args, str):
74 element = pythonType(*args, **initDict)
75 except Exception
as e:
76 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
77 raise type(e)(message, e.args)
82 """Define and execute a calculation on a ParquetTable
84 The `__call__` method accepts either a `ParquetTable` object or a
85 `DeferredDatasetHandle`,
and returns the
86 result of the calculation
as a single column. Each functor defines what
87 columns are needed
for the calculation,
and only these columns are read
88 from the `ParquetTable`.
90 The action of `__call__` consists of two steps: first, loading the
91 necessary columns
from disk into memory
as a `pandas.DataFrame` object;
92 and second, performing the computation on this dataframe
and returning the
96 To define a new `Functor`, a subclass must define a `_func` method,
97 that takes a `pandas.DataFrame`
and returns result
in a `pandas.Series`.
98 In addition, it must define the following attributes
100 * `_columns`: The columns necessary to perform the calculation
101 * `name`: A name appropriate
for a figure axis label
102 * `shortname`: A name appropriate
for use
as a dictionary key
104 On initialization, a `Functor` should declare what band (`filt` kwarg)
105 and dataset (e.g. `
'ref'`, `
'meas'`, `
'forced_src'`) it
is intended to be
106 applied to. This enables the `_get_data` method to extract the proper
107 columns
from the parquet file. If
not specified, the dataset will fall back
108 on the `_defaultDataset`attribute. If band
is not specified
and `dataset`
109 is anything other than `
'ref'`, then an error will be raised when trying to
110 perform the calculation.
112 Originally, `Functor` was set up to expect
113 datasets formatted like the `deepCoadd_obj` dataset; that
is, a
114 dataframe
with a multi-level column index,
with the levels of the
115 column index being `band`, `dataset`,
and `column`.
116 It has since been generalized to apply to dataframes without mutli-level
117 indices
and multi-level indices
with just `dataset`
and `column` levels.
118 In addition, the `_get_data` method that reads
119 the dataframe
from the `ParquetTable` will
return a dataframe
with column
120 index levels defined by the `_dfLevels` attribute; by default, this
is
123 The `_dfLevels` attributes should generally
not need to
124 be changed, unless `_func` needs columns
from multiple filters
or datasets
125 to do the calculation.
127 which `_dfLevels = (
'band',
'column')`,
and `_func` expects the dataframe
128 it gets to have those levels
in the column index.
133 Filter upon which to do the calculation
136 Dataset upon which to do the calculation
137 (e.g.,
'ref',
'meas',
'forced_src').
141 _defaultDataset = 'ref'
142 _dfLevels = (
'column',)
143 _defaultNoDup =
False
145 def __init__(self, filt=None, dataset=None, noDup=None):
149 self.
log = logging.getLogger(type(self).__name__)
153 if self.
_noDup is not None:
160 """Columns required to perform calculation
162 if not hasattr(self,
'_columns'):
163 raise NotImplementedError(
'Must define columns property or _columns attribute')
166 def _get_data_columnLevels(self, data, columnIndex=None):
167 """Gets the names of the column index levels
169 This should only be called in the context of a multilevel table.
170 The logic here
is to enable this to work both
with the gen2 `MultilevelParquetTable`
171 and with the gen3 `DeferredDatasetHandle`.
175 data : `MultilevelParquetTable`
or `DeferredDatasetHandle`
177 columnnIndex (optional): pandas `Index` object
178 if not passed, then it
is read
from the `DeferredDatasetHandle`
180 if isinstance(data, DeferredDatasetHandle):
181 if columnIndex
is None:
182 columnIndex = data.get(component=
"columns")
183 if columnIndex
is not None:
184 return columnIndex.names
185 if isinstance(data, MultilevelParquetTable):
186 return data.columnLevels
188 raise TypeError(f
"Unknown type for data: {type(data)}!")
190 def _get_data_columnLevelNames(self, data, columnIndex=None):
191 """Gets the content of each of the column levels for a multilevel table
193 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
197 if isinstance(data, DeferredDatasetHandle):
198 if columnIndex
is None:
199 columnIndex = data.get(component=
"columns")
200 if columnIndex
is not None:
201 columnLevels = columnIndex.names
203 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
204 for i, level
in enumerate(columnLevels)
206 return columnLevelNames
207 if isinstance(data, MultilevelParquetTable):
208 return data.columnLevelNames
210 raise TypeError(f
"Unknown type for data: {type(data)}!")
212 def _colsFromDict(self, colDict, columnIndex=None):
213 """Converts dictionary column specficiation to a list of columns
220 for i, lev
in enumerate(columnLevels):
222 if isinstance(colDict[lev], str):
223 new_colDict[lev] = [colDict[lev]]
225 new_colDict[lev] = colDict[lev]
227 new_colDict[lev] = columnIndex.levels[i]
229 levelCols = [new_colDict[lev]
for lev
in columnLevels]
230 cols = list(product(*levelCols))
231 colsAvailable = [col
for col
in cols
if col
in columnIndex]
235 """Returns columns needed by functor from multilevel dataset
237 To access tables with multilevel column structure, the `MultilevelParquetTable`
238 or `DeferredDatasetHandle` need to be passed either a list of tuples
or a
243 data : `MultilevelParquetTable`
or `DeferredDatasetHandle`
245 columnIndex (optional): pandas `Index` object
246 either passed
or read
in from `DeferredDatasetHandle`.
249 If true, then
return a list of tuples rather than the column dictionary
250 specification. This
is set to `
True` by `CompositeFunctor`
in order to be able to
251 combine columns
from the various component functors.
254 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
255 columnIndex = data.get(component=
"columns")
260 columnDict = {
'column': self.
columns,
262 if self.
filt is None:
264 if "band" in columnLevels:
266 columnDict[
"band"] = columnLevelNames[
"band"][0]
268 raise ValueError(f
"'filt' not set for functor {self.name}"
269 f
"(dataset {self.dataset}) "
271 "contains multiple filters in column index. "
272 "Set 'filt' or set 'dataset' to 'ref'.")
274 columnDict[
'band'] = self.
filt
276 if isinstance(data, MultilevelParquetTable):
277 return data._colsFromDict(columnDict)
278 elif isinstance(data, DeferredDatasetHandle):
280 return self.
_colsFromDict(columnDict, columnIndex=columnIndex)
284 def _func(self, df, dropna=True):
285 raise NotImplementedError(
'Must define calculation on dataframe')
287 def _get_columnIndex(self, data):
288 """Return columnIndex
291 if isinstance(data, DeferredDatasetHandle):
292 return data.get(component=
"columns")
296 def _get_data(self, data):
297 """Retrieve dataframe necessary for calculation.
299 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
301 Returns dataframe upon which `self.
_func` can act.
303 N.B.
while passing a raw pandas `DataFrame` *should* work here, it has
not been tested.
305 if isinstance(data, pd.DataFrame):
310 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
313 if isinstance(data, ParquetTable)
and not is_multiLevel:
315 df = data.toDataFrame(columns=columns)
324 if isinstance(data, MultilevelParquetTable):
326 df = data.toDataFrame(columns=columns, droplevels=
False)
327 elif isinstance(data, DeferredDatasetHandle):
329 df = data.get(parameters={
"columns": columns})
337 def _setLevels(self, df):
338 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels]
339 df.columns = df.columns.droplevel(levelsToDrop)
342 def _dropna(self, vals):
348 vals = self.
_func(df)
349 except Exception
as e:
350 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
358 """Computes difference between functor called on two different ParquetTable objects
360 return self(data1, **kwargs) - self(data2, **kwargs)
363 return pd.Series(np.full(len(df), np.nan), index=df.index)
367 """Full name of functor (suitable for figure labels)
369 return NotImplementedError
373 """Short name of functor (suitable for column name/dict key)
379 """Perform multiple calculations at once on a catalog
381 The role of a `CompositeFunctor` is to group together computations
from
382 multiple functors. Instead of returning `pandas.Series` a
383 `CompositeFunctor` returns a `pandas.Dataframe`,
with the column names
384 being the keys of `funcDict`.
386 The `columns` attribute of a `CompositeFunctor`
is the union of all columns
387 in all the component functors.
389 A `CompositeFunctor` does
not use a `_func` method itself; rather,
390 when a `CompositeFunctor`
is called, all its columns are loaded
391 at once,
and the resulting dataframe
is passed to the `_func` method of each component
392 functor. This has the advantage of only doing I/O (reading
from parquet file) once,
393 and works because each individual `_func` method of each component functor does
not
394 care
if there are *extra* columns
in the dataframe being passed; only that it must contain
395 *at least* the `columns` it expects.
397 An important
and useful
class method is `from_yaml`, which takes
as argument the path to a YAML
398 file specifying a collection of functors.
402 funcs : `dict`
or `list`
403 Dictionary
or list of functors. If a list, then it will be converted
404 into a dictonary according to the `.shortname` attribute of each functor.
411 if type(funcs) == dict:
414 self.
funcDict = {f.shortname: f
for f
in funcs}
432 if isinstance(new, dict):
434 elif isinstance(new, CompositeFunctor):
437 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
445 return list(set([x
for y
in [f.columns
for f
in self.
funcDict.values()]
for x
in y]))
454 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDict.values()
462 """Apply the functor to the data table
466 data : `lsst.daf.butler.DeferredDatasetHandle`,
469 or `pandas.DataFrame`.
470 The table
or a pointer to a table on disk
from which columns can
476 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
482 if isinstance(data, MultilevelParquetTable):
484 df = data.toDataFrame(columns=columns, droplevels=
False)
485 elif isinstance(data, DeferredDatasetHandle):
487 df = data.get(parameters={
"columns": columns})
492 subdf = f._setLevels(
493 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
495 valDict[k] = f._func(subdf)
496 except Exception
as e:
497 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
499 valDict[k] = f.fail(subdf)
504 if isinstance(data, DeferredDatasetHandle):
507 elif isinstance(data, pd.DataFrame):
514 valDict = {k: f._func(df)
for k, f
in self.
funcDict.items()}
517 for name, colVal
in valDict.items():
518 if len(colVal.shape) != 1:
519 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
520 "It is shaped %s and type %s." % (name, colVal.shape, type(colVal)))
523 valDf = pd.concat(valDict, axis=1)
525 print([(k, type(v))
for k, v
in valDict.items()])
528 if kwargs.get(
'dropna',
False):
529 valDf = valDf.dropna(how=
'any')
535 if renameRules
is None:
537 for old, new
in renameRules:
538 if col.startswith(old):
539 col = col.replace(old, new)
545 filename = os.path.expandvars(filename)
546 with open(filename)
as f:
547 translationDefinition = yaml.safe_load(f)
549 return cls.
from_yaml(translationDefinition, **kwargs)
554 for func, val
in translationDefinition[
'funcs'].items():
557 if 'flag_rename_rules' in translationDefinition:
558 renameRules = translationDefinition[
'flag_rename_rules']
562 if 'calexpFlags' in translationDefinition:
563 for flag
in translationDefinition[
'calexpFlags']:
564 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
566 if 'refFlags' in translationDefinition:
567 for flag
in translationDefinition[
'refFlags']:
570 if 'forcedFlags' in translationDefinition:
571 for flag
in translationDefinition[
'forcedFlags']:
572 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
574 if 'flags' in translationDefinition:
575 for flag
in translationDefinition[
'flags']:
578 return cls(funcs, **kwargs)
582 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
584 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
588 df : pandas.DataFrame
589 Dataframe on which to evaluate expression.
595 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
596 val = df.eval(expr_new)
597 except Exception
as e:
598 log.error(
"Exception in mag_aware_eval: %s: %s", type(e).__name__, e)
599 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
600 val = df.eval(expr_new)
605 """Arbitrary computation on a catalog
607 Column names (and thus the columns to be loaded
from catalog) are found
608 by finding all words
and trying to ignore all
"math-y" words.
613 Expression to evaluate, to be parsed
and executed by `mag_aware_eval`.
615 _ignore_words = ('mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
627 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
expr)
629 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
expr)
if c
not in self.
_ignore_words]
632 if not re.search(
'_instFlux$', c):
633 cols.append(f
'{c}_instFlux')
638 return list(set([c
for c
in cols
if c
not in not_a_col]))
645 """Get column with specified name
665 """Return the value of the index for each object
668 columns = ['coord_ra']
669 _defaultDataset =
'ref'
673 return pd.Series(df.index, index=df.index)
678 _allow_difference =
False
682 return pd.Series(df.index, index=df.index)
686 col =
'base_Footprint_nPix'
690 """Base class for coordinate column, in degrees
699 output = df[self.
col] * 180 / np.pi
if self.
_radians else df[self.
col]
704 """Right Ascension, in degrees
710 super().
__init__(
'coord_ra', **kwargs)
713 return super().
__call__(catalog, **kwargs)
717 """Declination, in degrees
723 super().
__init__(
'coord_dec', **kwargs)
726 return super().
__call__(catalog, **kwargs)
730 """Compute the level 20 HtmIndex for the catalog.
734 This functor was implemented to satisfy requirements of old APDB interface
735 which required ``pixelId`` column in DiaObject
with HTM20 index. APDB
736 interface had migrated to
not need that information, but we keep this
737 class in case it may be useful for something else.
752 def computePixel(row):
761 return self.
pixelator.index(sphPoint.getVector())
763 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
767 if not col.endswith(
'_instFlux'):
773 if not col.endswith(
'_instFluxErr'):
774 col +=
'_instFluxErr'
779 """Compute calibrated magnitude
781 Takes a `calib` argument, which returns the flux at mag=0
782 as `calib.getFluxMag0()`. If
not provided, then the default
783 `fluxMag0`
is 63095734448.0194, which
is default
for HSC.
784 This default should be removed
in DM-21955
786 This calculation hides warnings about invalid values
and dividing by zero.
788 As
for all functors, a `dataset`
and `filt` kwarg should be provided upon
789 initialization. Unlike the default `Functor`, however, the default dataset
790 for a `Mag`
is `
'meas'`, rather than `
'ref'`.
795 Name of flux column
from which to compute magnitude. Can be parseable
796 by `lsst.pipe.tasks.functors.fluxName` function---that
is, you can
pass
797 `
'modelfit_CModel'` instead of `
'modelfit_CModel_instFlux'`)
and it will
799 calib : `lsst.afw.image.calib.Calib` (optional)
800 Object that knows zero point.
802 _defaultDataset = 'meas'
807 if calib
is not None:
820 with np.warnings.catch_warnings():
821 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
822 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
827 return f
'mag_{self.col}'
831 """Compute calibrated magnitude uncertainty
838 calib : `lsst.afw.image.calib.Calib` (optional)
839 Object that knows zero point.
844 if self.
calib is not None:
851 return [self.
col, self.
col +
'Err']
854 with np.warnings.catch_warnings():
855 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
856 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
858 x = df[fluxErrCol] / df[fluxCol]
860 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
865 return super().name +
'_err'
877 _defaultDataset =
'meas'
879 """Functor to calculate magnitude difference"""
891 with np.warnings.catch_warnings():
892 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
893 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
894 return -2.5*np.log10(df[self.
col1]/df[self.
col2])
898 return f
'(mag_{self.col1} - mag_{self.col2})'
902 return f
'magDiff_{self.col1}_{self.col2}'
906 """Compute the color between two filters
908 Computes color by initializing two different `Mag`
909 functors based on the `col` and filters provided,
and
910 then returning the difference.
912 This
is enabled by the `_func` expecting a dataframe
with a
913 multilevel column index,
with both `
'band'`
and `
'column'`,
914 instead of just `
'column'`, which
is the `Functor` default.
915 This
is controlled by the `_dfLevels` attribute.
917 Also of note, the default dataset
for `Color`
is `forced_src
'`,
918 whereas for `Mag` it
is `
'meas'`.
923 Name of flux column
from which to compute; same
as would be passed to
927 Filters
from which to compute magnitude difference.
928 Color computed
is `
Mag(filt2) -
Mag(filt1)`.
930 _defaultDataset = 'forced_src'
931 _dfLevels = (
'band',
'column')
937 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
955 mag2 = self.mag2._func(df[self.filt2])
956 mag1 = self.mag1._func(df[self.filt1])
961 return [self.
mag1.col, self.
mag2.col]
968 return f
'{self.filt2} - {self.filt1} ({self.col})'
972 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
976 """Main function of this subclass is to override the dropna=True
979 _allow_difference =
False
984 return super().
__call__(parq, dropna=
False, **kwargs)
988 _columns = [
"base_ClassificationExtendedness_value"]
989 _column =
"base_ClassificationExtendedness_value"
994 test = (x < 0.5).astype(int)
995 test = test.mask(mask, 2)
1000 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
1001 index=x.index, name=
'label')
1003 label = label.astype(str)
1008 _columns = [
'numStarFlags']
1009 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
1011 def _func(self, df):
1015 n = len(x.unique()) - 1
1017 labels = [
'noStar',
'maybe',
'star']
1018 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
1019 index=x.index, name=
'label')
1022 label = label.astype(str)
1028 name =
'Deconvolved Moments'
1029 shortname =
'deconvolvedMoments'
1030 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1031 "ext_shapeHSM_HsmSourceMoments_yy",
1032 "base_SdssShape_xx",
"base_SdssShape_yy",
1033 "ext_shapeHSM_HsmPsfMoments_xx",
1034 "ext_shapeHSM_HsmPsfMoments_yy")
1036 def _func(self, df):
1037 """Calculate deconvolved moments"""
1038 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1039 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1041 hsm = np.ones(len(df))*np.nan
1042 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1043 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1044 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1049 raise RuntimeError(
'No psf shape parameter found in catalog')
1051 return hsm.where(np.isfinite(hsm), sdss) - psf
1055 """Functor to calculate SDSS trace radius size for sources"""
1056 name =
"SDSS Trace Size"
1057 shortname =
'sdssTrace'
1058 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1060 def _func(self, df):
1061 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1066 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1067 name =
"PSF - SDSS Trace Size"
1068 shortname =
'psf_sdssTrace'
1069 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1070 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1072 def _func(self, df):
1073 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1074 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1075 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1080 """Functor to calculate HSM trace radius size for sources"""
1081 name =
'HSM Trace Size'
1082 shortname =
'hsmTrace'
1083 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1084 "ext_shapeHSM_HsmSourceMoments_yy")
1086 def _func(self, df):
1087 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1088 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1093 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1094 name =
'PSF - HSM Trace Size'
1095 shortname =
'psf_HsmTrace'
1096 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1097 "ext_shapeHSM_HsmSourceMoments_yy",
1098 "ext_shapeHSM_HsmPsfMoments_xx",
1099 "ext_shapeHSM_HsmPsfMoments_yy")
1101 def _func(self, df):
1102 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1103 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1104 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1105 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1106 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1111 name =
'HSM Psf FWHM'
1112 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1115 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1117 def _func(self, df):
1119 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1123 name =
"Distortion Ellipticity (e1)"
1124 shortname =
"Distortion"
1137 def _func(self, df):
1142 name =
"Ellipticity e2"
1154 def _func(self, df):
1170 def _func(self, df):
1175 """Computations using the stored localWcs.
1177 name = "LocalWcsOperations"
1192 """Compute the distance on the sphere from x2, y1 to x1, y1.
1200 cd11 : `pandas.Series`
1201 [1, 1] element of the local Wcs affine transform.
1202 cd11 : `pandas.Series`
1203 [1, 1] element of the local Wcs affine transform.
1204 cd12 : `pandas.Series`
1205 [1, 2] element of the local Wcs affine transform.
1206 cd21 : `pandas.Series`
1207 [2, 1] element of the local Wcs affine transform.
1208 cd22 : `pandas.Series`
1209 [2, 2] element of the local Wcs affine transform.
1214 RA and dec conversion of x
and y given the local Wcs. Returned
1215 units are
in radians.
1218 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1221 """Compute the local pixel scale conversion.
1225 ra1 : `pandas.Series`
1226 Ra of the first coordinate in radians.
1227 dec1 : `pandas.Series`
1228 Dec of the first coordinate
in radians.
1229 ra2 : `pandas.Series`
1230 Ra of the second coordinate
in radians.
1231 dec2 : `pandas.Series`
1232 Dec of the second coordinate
in radians.
1236 dist : `pandas.Series`
1237 Distance on the sphere
in radians.
1239 deltaDec = dec2 - dec1
1241 return 2 * np.arcsin(
1243 np.sin(deltaDec / 2) ** 2
1244 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1247 """Compute the distance on the sphere from x2, y1 to x1, y1.
1251 x1 : `pandas.Series`
1253 y1 : `pandas.Series`
1255 x2 : `pandas.Series`
1257 y2 : `pandas.Series`
1259 cd11 : `pandas.Series`
1260 [1, 1] element of the local Wcs affine transform.
1261 cd11 : `pandas.Series`
1262 [1, 1] element of the local Wcs affine transform.
1263 cd12 : `pandas.Series`
1264 [1, 2] element of the local Wcs affine transform.
1265 cd21 : `pandas.Series`
1266 [2, 1] element of the local Wcs affine transform.
1267 cd22 : `pandas.Series`
1268 [2, 2] element of the local Wcs affine transform.
1272 Distance : `pandas.Series`
1273 Arcseconds per pixel at the location of the local WC
1282 """Compute the local pixel scale from the stored CDMatrix.
1294 """Compute the local pixel to scale conversion in arcseconds.
1298 cd11 : `pandas.Series`
1299 [1, 1] element of the local Wcs affine transform in radians.
1300 cd11 : `pandas.Series`
1301 [1, 1] element of the local Wcs affine transform
in radians.
1302 cd12 : `pandas.Series`
1303 [1, 2] element of the local Wcs affine transform
in radians.
1304 cd21 : `pandas.Series`
1305 [2, 1] element of the local Wcs affine transform
in radians.
1306 cd22 : `pandas.Series`
1307 [2, 2] element of the local Wcs affine transform
in radians.
1311 pixScale : `pandas.Series`
1312 Arcseconds per pixel at the location of the local WC
1314 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1316 def _func(self, df):
1324 """Convert a value in units pixels squared to units arcseconds squared.
1343 return f
"{self.col}_asArcseconds"
1353 def _func(self, df):
1361 """Convert a value in units pixels to units arcseconds.
1380 return f
"{self.col}_asArcsecondsSq"
1390 def _func(self, df):
1395 return df[self.
col] * pixScale * pixScale
1399 name =
'Reference Band'
1400 shortname =
'refBand'
1404 return [
"merge_measurement_i",
1405 "merge_measurement_r",
1406 "merge_measurement_z",
1407 "merge_measurement_y",
1408 "merge_measurement_g",
1409 "merge_measurement_u"]
1411 def _func(self, df: pd.DataFrame) -> pd.Series:
1412 def getFilterAliasName(row):
1414 colName = row.idxmax()
1415 return colName.replace(
'merge_measurement_',
'')
1419 columns = [col
for col
in self.
columnscolumns if col
in df.columns]
1421 return df[columns].apply(getFilterAliasName, axis=1,
1422 result_type=
'reduce').astype(
'object')
1427 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1428 LOG_AB_FLUX_SCALE = 12.56
1429 FIVE_OVER_2LOG10 = 1.085736204758129569
1433 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1439 if calib
is not None:
1453 return f
'mag_{self.col}'
1457 if np.abs(a) < np.abs(b):
1462 return np.abs(a) * np.sqrt(1. + q*q)
1468 with np.warnings.catch_warnings():
1469 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1470 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1471 return -2.5 * np.log10(dn/fluxMag0)
1474 retVal = self.
vhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1479 retVal = self.
dn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2flux(dn, fluxMag0)
1484 def _func(self, df):
1493 def _func(self, df):
1495 return pd.Series(retArr, index=df.index)
1499 def _func(self, df):
1508 def _func(self, df):
1510 return pd.Series(retArr, index=df.index)
1514 """Base class for calibrating the specified instrument flux column using
1515 the local photometric calibration.
1520 Name of the instrument flux column.
1521 instFluxErrCol : `str`
1522 Name of the assocated error columns for ``instFluxCol``.
1523 photoCalibCol : `str`
1524 Name of local calibration column.
1525 photoCalibErrCol : `str`
1526 Error associated
with ``photoCalibCol``
1536 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1551 """Convert instrument flux to nanojanskys.
1555 instFlux : `numpy.ndarray` or `pandas.Series`
1556 Array of instrument flux measurements
1557 localCalib : `numpy.ndarray`
or `pandas.Series`
1558 Array of local photometric calibration estimates.
1562 calibFlux : `numpy.ndarray`
or `pandas.Series`
1563 Array of calibrated flux measurements.
1565 return instFlux * localCalib
1568 """Convert instrument flux to nanojanskys.
1572 instFlux : `numpy.ndarray` or `pandas.Series`
1573 Array of instrument flux measurements
1574 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1575 Errors on associated ``instFlux`` values
1576 localCalib : `numpy.ndarray`
or `pandas.Series`
1577 Array of local photometric calibration estimates.
1578 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1579 Errors on associated ``localCalib`` values
1583 calibFluxErr : `numpy.ndarray`
or `pandas.Series`
1584 Errors on calibrated flux measurements.
1586 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1589 """Convert instrument flux to nanojanskys.
1593 instFlux : `numpy.ndarray` or `pandas.Series`
1594 Array of instrument flux measurements
1595 localCalib : `numpy.ndarray`
or `pandas.Series`
1596 Array of local photometric calibration estimates.
1600 calibMag : `numpy.ndarray`
or `pandas.Series`
1601 Array of calibrated AB magnitudes.
1606 """Convert instrument flux err to nanojanskys.
1610 instFlux : `numpy.ndarray` or `pandas.Series`
1611 Array of instrument flux measurements
1612 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1613 Errors on associated ``instFlux`` values
1614 localCalib : `numpy.ndarray`
or `pandas.Series`
1615 Array of local photometric calibration estimates.
1616 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1617 Errors on associated ``localCalib`` values
1621 calibMagErr: `numpy.ndarray`
or `pandas.Series`
1622 Error on calibrated AB magnitudes.
1629 """Compute calibrated fluxes using the local calibration value.
1645 return f
'flux_{self.instFluxCol}'
1647 def _func(self, df):
1652 """Compute calibrated flux errors using the local calibration value.
1669 return f
'fluxErr_{self.instFluxCol}'
1671 def _func(self, df):
1677 """Compute calibrated AB magnitudes using the local calibration value.
1693 return f
'mag_{self.instFluxCol}'
1695 def _func(self, df):
1701 """Compute calibrated AB magnitude errors using the local calibration value.
1718 return f
'magErr_{self.instFluxCol}'
1720 def _func(self, df):
1728 """Compute absolute mean of dipole fluxes.
1737 LocalDipoleMeanFluxErr
1739 LocalDipoleDiffFluxErr
1769 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1771 def _func(self, df):
1777 """Compute the error on the absolute mean of dipole fluxes.
1786 LocalDipoleMeanFluxErr
1788 LocalDipoleDiffFluxErr
1802 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1804 def _func(self, df):
1813 """Compute the absolute difference of dipole fluxes.
1815 Value is (abs(pos) - abs(neg))
1824 LocalDipoleMeanFluxErr
1826 LocalDipoleDiffFluxErr
1837 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1839 def _func(self, df):
1845 """Compute the error on the absolute difference of dipole fluxes.
1854 LocalDipoleMeanFluxErr
1856 LocalDipoleDiffFluxErr
1870 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1872 def _func(self, df):
1881 """Base class for returning the ratio of 2 columns.
1883 Can be used to compute a Signal to Noise ratio for any input flux.
1888 Name of the column to use at the numerator
in the ratio
1890 Name of the column to use
as the denominator
in the ratio.
1906 return f
'ratio_{self.numerator}_{self.denominator}'
1908 def _func(self, df):
1909 with np.warnings.catch_warnings():
1910 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1911 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1916 """Compute E(B-V) from dustmaps.sfd
1918 _defaultDataset = 'ref'
1923 self.
_columns = [
'coord_ra',
'coord_dec']
1927 def _func(self, df):
1928 coords = SkyCoord(df[
'coord_ra']*u.rad, df[
'coord_dec']*u.rad)
1929 ebv = self.
sfd(coords)
1932 return pd.Series(ebv, index=df.index).astype(
'float64')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr, log)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)