22__all__ = [
"Functor",
"CompositeFunctor",
"CustomFunctor",
"Column",
"Index",
23 "IDColumn",
"FootprintNPix",
"CoordColumn",
"RAColumn",
"DecColumn",
24 "HtmIndex20",
"Mag",
"MagErr",
"NanoMaggie",
"MagDiff",
"Color",
25 "Labeller",
"StarGalaxyLabeller",
"NumStarLabeller",
"DeconvolvedMoments",
26 "SdssTraceSize",
"PsfSdssTraceSizeDiff",
"HsmTraceSize",
"PsfHsmTraceSizeDiff",
27 "HsmFwhm",
"E1",
"E2",
"RadiusFromQuadrupole",
"LocalWcs",
"ComputePixelScale",
28 "ConvertPixelToArcseconds",
"ConvertPixelSqToArcsecondsSq",
"ReferenceBand",
29 "Photometry",
"NanoJansky",
"NanoJanskyErr",
"Magnitude",
"MagnitudeErr",
30 "LocalPhotometry",
"LocalNanojansky",
"LocalNanojanskyErr",
31 "LocalMagnitude",
"LocalMagnitudeErr",
"LocalDipoleMeanFlux",
32 "LocalDipoleMeanFluxErr",
"LocalDipoleDiffFlux",
"LocalDipoleDiffFluxErr",
37from itertools
import product
43import astropy.units
as u
44from astropy.coordinates
import SkyCoord
46from lsst.utils
import doImport
47from lsst.daf.butler
import DeferredDatasetHandle
51from .parquetTable
import ParquetTable, MultilevelParquetTable
55 typeKey='functor', name=None):
56 """Initialize an object defined in a dictionary
58 The object needs to be importable as
59 f
'{basePath}.{initDict[typeKey]}'
60 The positional
and keyword arguments (
if any) are contained
in
61 "args" and "kwargs" entries
in the dictionary, respectively.
62 This
is used
in `functors.CompositeFunctor.from_yaml` to initialize
63 a composite functor
from a specification
in a YAML file.
68 Dictionary describing object
's initialization. Must contain
69 an entry keyed by ``typeKey`` that is the name of the object,
70 relative to ``basePath``.
72 Path relative to module
in which ``initDict[typeKey]``
is defined.
74 Key of ``initDict`` that
is the name of the object
75 (relative to `basePath`).
77 initDict = initDict.copy()
79 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
81 if 'args' in initDict:
82 args = initDict.pop(
'args')
83 if isinstance(args, str):
86 element = pythonType(*args, **initDict)
87 except Exception
as e:
88 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
89 raise type(e)(message, e.args)
94 """Define and execute a calculation on a ParquetTable
96 The `__call__` method accepts either a `ParquetTable` object or a
97 `DeferredDatasetHandle`,
and returns the
98 result of the calculation
as a single column. Each functor defines what
99 columns are needed
for the calculation,
and only these columns are read
100 from the `ParquetTable`.
102 The action of `__call__` consists of two steps: first, loading the
103 necessary columns
from disk into memory
as a `pandas.DataFrame` object;
104 and second, performing the computation on this dataframe
and returning the
108 To define a new `Functor`, a subclass must define a `_func` method,
109 that takes a `pandas.DataFrame`
and returns result
in a `pandas.Series`.
110 In addition, it must define the following attributes
112 * `_columns`: The columns necessary to perform the calculation
113 * `name`: A name appropriate
for a figure axis label
114 * `shortname`: A name appropriate
for use
as a dictionary key
116 On initialization, a `Functor` should declare what band (`filt` kwarg)
117 and dataset (e.g. `
'ref'`, `
'meas'`, `
'forced_src'`) it
is intended to be
118 applied to. This enables the `_get_data` method to extract the proper
119 columns
from the parquet file. If
not specified, the dataset will fall back
120 on the `_defaultDataset`attribute. If band
is not specified
and `dataset`
121 is anything other than `
'ref'`, then an error will be raised when trying to
122 perform the calculation.
124 Originally, `Functor` was set up to expect
125 datasets formatted like the `deepCoadd_obj` dataset; that
is, a
126 dataframe
with a multi-level column index,
with the levels of the
127 column index being `band`, `dataset`,
and `column`.
128 It has since been generalized to apply to dataframes without mutli-level
129 indices
and multi-level indices
with just `dataset`
and `column` levels.
130 In addition, the `_get_data` method that reads
131 the dataframe
from the `ParquetTable` will
return a dataframe
with column
132 index levels defined by the `_dfLevels` attribute; by default, this
is
135 The `_dfLevels` attributes should generally
not need to
136 be changed, unless `_func` needs columns
from multiple filters
or datasets
137 to do the calculation.
139 which `_dfLevels = (
'band',
'column')`,
and `_func` expects the dataframe
140 it gets to have those levels
in the column index.
145 Filter upon which to do the calculation
148 Dataset upon which to do the calculation
149 (e.g.,
'ref',
'meas',
'forced_src').
153 _defaultDataset = 'ref'
154 _dfLevels = (
'column',)
155 _defaultNoDup =
False
157 def __init__(self, filt=None, dataset=None, noDup=None):
161 self.
log = logging.getLogger(type(self).__name__)
165 if self.
_noDup is not None:
172 """Columns required to perform calculation
174 if not hasattr(self,
'_columns'):
175 raise NotImplementedError(
'Must define columns property or _columns attribute')
178 def _get_data_columnLevels(self, data, columnIndex=None):
179 """Gets the names of the column index levels
181 This should only be called in the context of a multilevel table.
182 The logic here
is to enable this to work both
with the gen2 `MultilevelParquetTable`
183 and with the gen3 `DeferredDatasetHandle`.
187 data : `MultilevelParquetTable`
or `DeferredDatasetHandle`
189 columnnIndex (optional): pandas `Index` object
190 if not passed, then it
is read
from the `DeferredDatasetHandle`
192 if isinstance(data, DeferredDatasetHandle):
193 if columnIndex
is None:
194 columnIndex = data.get(component=
"columns")
195 if columnIndex
is not None:
196 return columnIndex.names
197 if isinstance(data, MultilevelParquetTable):
198 return data.columnLevels
200 raise TypeError(f
"Unknown type for data: {type(data)}!")
202 def _get_data_columnLevelNames(self, data, columnIndex=None):
203 """Gets the content of each of the column levels for a multilevel table
205 Similar to `_get_data_columnLevels`, this enables backward compatibility with gen2.
209 if isinstance(data, DeferredDatasetHandle):
210 if columnIndex
is None:
211 columnIndex = data.get(component=
"columns")
212 if columnIndex
is not None:
213 columnLevels = columnIndex.names
215 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
216 for i, level
in enumerate(columnLevels)
218 return columnLevelNames
219 if isinstance(data, MultilevelParquetTable):
220 return data.columnLevelNames
222 raise TypeError(f
"Unknown type for data: {type(data)}!")
224 def _colsFromDict(self, colDict, columnIndex=None):
225 """Converts dictionary column specficiation to a list of columns
232 for i, lev
in enumerate(columnLevels):
234 if isinstance(colDict[lev], str):
235 new_colDict[lev] = [colDict[lev]]
237 new_colDict[lev] = colDict[lev]
239 new_colDict[lev] = columnIndex.levels[i]
241 levelCols = [new_colDict[lev]
for lev
in columnLevels]
242 cols = list(product(*levelCols))
243 colsAvailable = [col
for col
in cols
if col
in columnIndex]
247 """Returns columns needed by functor from multilevel dataset
249 To access tables with multilevel column structure, the `MultilevelParquetTable`
250 or `DeferredDatasetHandle` need to be passed either a list of tuples
or a
255 data : `MultilevelParquetTable`
or `DeferredDatasetHandle`
257 columnIndex (optional): pandas `Index` object
258 either passed
or read
in from `DeferredDatasetHandle`.
261 If true, then
return a list of tuples rather than the column dictionary
262 specification. This
is set to `
True` by `CompositeFunctor`
in order to be able to
263 combine columns
from the various component functors.
266 if isinstance(data, DeferredDatasetHandle)
and columnIndex
is None:
267 columnIndex = data.get(component=
"columns")
272 columnDict = {
'column': self.
columns,
274 if self.
filt is None:
276 if "band" in columnLevels:
278 columnDict[
"band"] = columnLevelNames[
"band"][0]
280 raise ValueError(f
"'filt' not set for functor {self.name}"
281 f
"(dataset {self.dataset}) "
283 "contains multiple filters in column index. "
284 "Set 'filt' or set 'dataset' to 'ref'.")
286 columnDict[
'band'] = self.
filt
288 if isinstance(data, MultilevelParquetTable):
289 return data._colsFromDict(columnDict)
290 elif isinstance(data, DeferredDatasetHandle):
292 return self.
_colsFromDict(columnDict, columnIndex=columnIndex)
296 def _func(self, df, dropna=True):
297 raise NotImplementedError(
'Must define calculation on dataframe')
299 def _get_columnIndex(self, data):
300 """Return columnIndex
303 if isinstance(data, DeferredDatasetHandle):
304 return data.get(component=
"columns")
308 def _get_data(self, data):
309 """Retrieve dataframe necessary for calculation.
311 The data argument can be a DataFrame, a ParquetTable instance, or a gen3 DeferredDatasetHandle
313 Returns dataframe upon which `self.
_func` can act.
315 N.B.
while passing a raw pandas `DataFrame` *should* work here, it has
not been tested.
317 if isinstance(data, pd.DataFrame):
322 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
325 if isinstance(data, ParquetTable)
and not is_multiLevel:
327 df = data.toDataFrame(columns=columns)
336 if isinstance(data, MultilevelParquetTable):
338 df = data.toDataFrame(columns=columns, droplevels=
False)
339 elif isinstance(data, DeferredDatasetHandle):
341 df = data.get(parameters={
"columns": columns})
349 def _setLevels(self, df):
350 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels]
351 df.columns = df.columns.droplevel(levelsToDrop)
354 def _dropna(self, vals):
360 vals = self.
_func(df)
361 except Exception
as e:
362 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
370 """Computes difference between functor called on two different ParquetTable objects
372 return self(data1, **kwargs) - self(data2, **kwargs)
375 return pd.Series(np.full(len(df), np.nan), index=df.index)
379 """Full name of functor (suitable for figure labels)
381 return NotImplementedError
385 """Short name of functor (suitable for column name/dict key)
391 """Perform multiple calculations at once on a catalog
393 The role of a `CompositeFunctor` is to group together computations
from
394 multiple functors. Instead of returning `pandas.Series` a
395 `CompositeFunctor` returns a `pandas.Dataframe`,
with the column names
396 being the keys of `funcDict`.
398 The `columns` attribute of a `CompositeFunctor`
is the union of all columns
399 in all the component functors.
401 A `CompositeFunctor` does
not use a `_func` method itself; rather,
402 when a `CompositeFunctor`
is called, all its columns are loaded
403 at once,
and the resulting dataframe
is passed to the `_func` method of each component
404 functor. This has the advantage of only doing I/O (reading
from parquet file) once,
405 and works because each individual `_func` method of each component functor does
not
406 care
if there are *extra* columns
in the dataframe being passed; only that it must contain
407 *at least* the `columns` it expects.
409 An important
and useful
class method is `from_yaml`, which takes
as argument the path to a YAML
410 file specifying a collection of functors.
414 funcs : `dict`
or `list`
415 Dictionary
or list of functors. If a list, then it will be converted
416 into a dictonary according to the `.shortname` attribute of each functor.
423 if type(funcs) == dict:
426 self.
funcDict = {f.shortname: f
for f
in funcs}
444 if isinstance(new, dict):
446 elif isinstance(new, CompositeFunctor):
449 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
457 return list(set([x
for y
in [f.columns
for f
in self.
funcDict.values()]
for x
in y]))
466 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDict.values()
474 """Apply the functor to the data table
478 data : `lsst.daf.butler.DeferredDatasetHandle`,
481 or `pandas.DataFrame`.
482 The table
or a pointer to a table on disk
from which columns can
488 is_multiLevel = isinstance(data, MultilevelParquetTable)
or isinstance(columnIndex, pd.MultiIndex)
494 if isinstance(data, MultilevelParquetTable):
496 df = data.toDataFrame(columns=columns, droplevels=
False)
497 elif isinstance(data, DeferredDatasetHandle):
499 df = data.get(parameters={
"columns": columns})
504 subdf = f._setLevels(
505 df[f.multilevelColumns(data, returnTuple=
True, columnIndex=columnIndex)]
507 valDict[k] = f._func(subdf)
508 except Exception
as e:
509 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
511 valDict[k] = f.fail(subdf)
516 if isinstance(data, DeferredDatasetHandle):
519 elif isinstance(data, pd.DataFrame):
526 valDict = {k: f._func(df)
for k, f
in self.
funcDict.items()}
529 for name, colVal
in valDict.items():
530 if len(colVal.shape) != 1:
531 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
532 "It is shaped %s and type %s." % (name, colVal.shape, type(colVal)))
535 valDf = pd.concat(valDict, axis=1)
537 print([(k, type(v))
for k, v
in valDict.items()])
540 if kwargs.get(
'dropna',
False):
541 valDf = valDf.dropna(how=
'any')
547 if renameRules
is None:
549 for old, new
in renameRules:
550 if col.startswith(old):
551 col = col.replace(old, new)
557 filename = os.path.expandvars(filename)
558 with open(filename)
as f:
559 translationDefinition = yaml.safe_load(f)
561 return cls.
from_yaml(translationDefinition, **kwargs)
566 for func, val
in translationDefinition[
'funcs'].items():
569 if 'flag_rename_rules' in translationDefinition:
570 renameRules = translationDefinition[
'flag_rename_rules']
574 if 'calexpFlags' in translationDefinition:
575 for flag
in translationDefinition[
'calexpFlags']:
576 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
578 if 'refFlags' in translationDefinition:
579 for flag
in translationDefinition[
'refFlags']:
582 if 'forcedFlags' in translationDefinition:
583 for flag
in translationDefinition[
'forcedFlags']:
584 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
586 if 'flags' in translationDefinition:
587 for flag
in translationDefinition[
'flags']:
590 return cls(funcs, **kwargs)
594 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
596 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
600 df : pandas.DataFrame
601 Dataframe on which to evaluate expression.
607 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
608 val = df.eval(expr_new)
609 except Exception
as e:
610 log.error(
"Exception in mag_aware_eval: %s: %s", type(e).__name__, e)
611 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
612 val = df.eval(expr_new)
617 """Arbitrary computation on a catalog
619 Column names (and thus the columns to be loaded
from catalog) are found
620 by finding all words
and trying to ignore all
"math-y" words.
625 Expression to evaluate, to be parsed
and executed by `mag_aware_eval`.
627 _ignore_words = ('mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
639 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
expr)
641 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
expr)
if c
not in self.
_ignore_words]
644 if not re.search(
'_instFlux$', c):
645 cols.append(f
'{c}_instFlux')
650 return list(set([c
for c
in cols
if c
not in not_a_col]))
657 """Get column with specified name
677 """Return the value of the index for each object
680 columns = ['coord_ra']
681 _defaultDataset =
'ref'
685 return pd.Series(df.index, index=df.index)
690 _allow_difference =
False
694 return pd.Series(df.index, index=df.index)
698 col =
'base_Footprint_nPix'
702 """Base class for coordinate column, in degrees
711 output = df[self.
col] * 180 / np.pi
if self.
_radians else df[self.
col]
716 """Right Ascension, in degrees
722 super().
__init__(
'coord_ra', **kwargs)
725 return super().
__call__(catalog, **kwargs)
729 """Declination, in degrees
735 super().
__init__(
'coord_dec', **kwargs)
738 return super().
__call__(catalog, **kwargs)
742 """Compute the level 20 HtmIndex for the catalog.
746 This functor was implemented to satisfy requirements of old APDB interface
747 which required ``pixelId`` column in DiaObject
with HTM20 index. APDB
748 interface had migrated to
not need that information, but we keep this
749 class in case it may be useful for something else.
764 def computePixel(row):
773 return self.
pixelator.index(sphPoint.getVector())
775 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
779 if not col.endswith(
'_instFlux'):
785 if not col.endswith(
'_instFluxErr'):
786 col +=
'_instFluxErr'
791 """Compute calibrated magnitude
793 Takes a `calib` argument, which returns the flux at mag=0
794 as `calib.getFluxMag0()`. If
not provided, then the default
795 `fluxMag0`
is 63095734448.0194, which
is default
for HSC.
796 This default should be removed
in DM-21955
798 This calculation hides warnings about invalid values
and dividing by zero.
800 As
for all functors, a `dataset`
and `filt` kwarg should be provided upon
801 initialization. Unlike the default `Functor`, however, the default dataset
802 for a `Mag`
is `
'meas'`, rather than `
'ref'`.
807 Name of flux column
from which to compute magnitude. Can be parseable
808 by `lsst.pipe.tasks.functors.fluxName` function---that
is, you can
pass
809 `
'modelfit_CModel'` instead of `
'modelfit_CModel_instFlux'`)
and it will
811 calib : `lsst.afw.image.calib.Calib` (optional)
812 Object that knows zero point.
814 _defaultDataset = 'meas'
819 if calib
is not None:
832 with np.warnings.catch_warnings():
833 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
834 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
839 return f
'mag_{self.col}'
843 """Compute calibrated magnitude uncertainty
850 calib : `lsst.afw.image.calib.Calib` (optional)
851 Object that knows zero point.
856 if self.
calib is not None:
863 return [self.
col, self.
col +
'Err']
866 with np.warnings.catch_warnings():
867 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
868 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
870 x = df[fluxErrCol] / df[fluxCol]
872 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
877 return super().name +
'_err'
889 _defaultDataset =
'meas'
891 """Functor to calculate magnitude difference"""
903 with np.warnings.catch_warnings():
904 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
905 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
906 return -2.5*np.log10(df[self.
col1]/df[self.
col2])
910 return f
'(mag_{self.col1} - mag_{self.col2})'
914 return f
'magDiff_{self.col1}_{self.col2}'
918 """Compute the color between two filters
920 Computes color by initializing two different `Mag`
921 functors based on the `col` and filters provided,
and
922 then returning the difference.
924 This
is enabled by the `_func` expecting a dataframe
with a
925 multilevel column index,
with both `
'band'`
and `
'column'`,
926 instead of just `
'column'`, which
is the `Functor` default.
927 This
is controlled by the `_dfLevels` attribute.
929 Also of note, the default dataset
for `Color`
is `forced_src
'`,
930 whereas for `Mag` it
is `
'meas'`.
935 Name of flux column
from which to compute; same
as would be passed to
939 Filters
from which to compute magnitude difference.
940 Color computed
is `
Mag(filt2) -
Mag(filt1)`.
942 _defaultDataset = 'forced_src'
943 _dfLevels = (
'band',
'column')
949 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
967 mag2 = self.mag2._func(df[self.filt2])
968 mag1 = self.mag1._func(df[self.filt1])
973 return [self.
mag1.col, self.
mag2.col]
980 return f
'{self.filt2} - {self.filt1} ({self.col})'
984 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
988 """Main function of this subclass is to override the dropna=True
991 _allow_difference =
False
996 return super().
__call__(parq, dropna=
False, **kwargs)
1000 _columns = [
"base_ClassificationExtendedness_value"]
1001 _column =
"base_ClassificationExtendedness_value"
1003 def _func(self, df):
1006 test = (x < 0.5).astype(int)
1007 test = test.mask(mask, 2)
1012 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
1013 index=x.index, name=
'label')
1015 label = label.astype(str)
1020 _columns = [
'numStarFlags']
1021 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
1023 def _func(self, df):
1027 n = len(x.unique()) - 1
1029 labels = [
'noStar',
'maybe',
'star']
1030 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
1031 index=x.index, name=
'label')
1034 label = label.astype(str)
1040 name =
'Deconvolved Moments'
1041 shortname =
'deconvolvedMoments'
1042 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1043 "ext_shapeHSM_HsmSourceMoments_yy",
1044 "base_SdssShape_xx",
"base_SdssShape_yy",
1045 "ext_shapeHSM_HsmPsfMoments_xx",
1046 "ext_shapeHSM_HsmPsfMoments_yy")
1048 def _func(self, df):
1049 """Calculate deconvolved moments"""
1050 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1051 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1053 hsm = np.ones(len(df))*np.nan
1054 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1055 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1056 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1061 raise RuntimeError(
'No psf shape parameter found in catalog')
1063 return hsm.where(np.isfinite(hsm), sdss) - psf
1067 """Functor to calculate SDSS trace radius size for sources"""
1068 name =
"SDSS Trace Size"
1069 shortname =
'sdssTrace'
1070 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1072 def _func(self, df):
1073 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1078 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1079 name =
"PSF - SDSS Trace Size"
1080 shortname =
'psf_sdssTrace'
1081 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1082 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1084 def _func(self, df):
1085 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1086 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1087 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1092 """Functor to calculate HSM trace radius size for sources"""
1093 name =
'HSM Trace Size'
1094 shortname =
'hsmTrace'
1095 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1096 "ext_shapeHSM_HsmSourceMoments_yy")
1098 def _func(self, df):
1099 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1100 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1105 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1106 name =
'PSF - HSM Trace Size'
1107 shortname =
'psf_HsmTrace'
1108 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1109 "ext_shapeHSM_HsmSourceMoments_yy",
1110 "ext_shapeHSM_HsmPsfMoments_xx",
1111 "ext_shapeHSM_HsmPsfMoments_yy")
1113 def _func(self, df):
1114 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1115 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1116 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1117 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1118 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1123 name =
'HSM Psf FWHM'
1124 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1127 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1129 def _func(self, df):
1131 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1135 name =
"Distortion Ellipticity (e1)"
1136 shortname =
"Distortion"
1149 def _func(self, df):
1154 name =
"Ellipticity e2"
1166 def _func(self, df):
1182 def _func(self, df):
1187 """Computations using the stored localWcs.
1189 name = "LocalWcsOperations"
1204 """Compute the distance on the sphere from x2, y1 to x1, y1.
1212 cd11 : `pandas.Series`
1213 [1, 1] element of the local Wcs affine transform.
1214 cd11 : `pandas.Series`
1215 [1, 1] element of the local Wcs affine transform.
1216 cd12 : `pandas.Series`
1217 [1, 2] element of the local Wcs affine transform.
1218 cd21 : `pandas.Series`
1219 [2, 1] element of the local Wcs affine transform.
1220 cd22 : `pandas.Series`
1221 [2, 2] element of the local Wcs affine transform.
1226 RA and dec conversion of x
and y given the local Wcs. Returned
1227 units are
in radians.
1230 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1233 """Compute the local pixel scale conversion.
1237 ra1 : `pandas.Series`
1238 Ra of the first coordinate in radians.
1239 dec1 : `pandas.Series`
1240 Dec of the first coordinate
in radians.
1241 ra2 : `pandas.Series`
1242 Ra of the second coordinate
in radians.
1243 dec2 : `pandas.Series`
1244 Dec of the second coordinate
in radians.
1248 dist : `pandas.Series`
1249 Distance on the sphere
in radians.
1251 deltaDec = dec2 - dec1
1253 return 2 * np.arcsin(
1255 np.sin(deltaDec / 2) ** 2
1256 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1259 """Compute the distance on the sphere from x2, y1 to x1, y1.
1263 x1 : `pandas.Series`
1265 y1 : `pandas.Series`
1267 x2 : `pandas.Series`
1269 y2 : `pandas.Series`
1271 cd11 : `pandas.Series`
1272 [1, 1] element of the local Wcs affine transform.
1273 cd11 : `pandas.Series`
1274 [1, 1] element of the local Wcs affine transform.
1275 cd12 : `pandas.Series`
1276 [1, 2] element of the local Wcs affine transform.
1277 cd21 : `pandas.Series`
1278 [2, 1] element of the local Wcs affine transform.
1279 cd22 : `pandas.Series`
1280 [2, 2] element of the local Wcs affine transform.
1284 Distance : `pandas.Series`
1285 Arcseconds per pixel at the location of the local WC
1294 """Compute the local pixel scale from the stored CDMatrix.
1306 """Compute the local pixel to scale conversion in arcseconds.
1310 cd11 : `pandas.Series`
1311 [1, 1] element of the local Wcs affine transform in radians.
1312 cd11 : `pandas.Series`
1313 [1, 1] element of the local Wcs affine transform
in radians.
1314 cd12 : `pandas.Series`
1315 [1, 2] element of the local Wcs affine transform
in radians.
1316 cd21 : `pandas.Series`
1317 [2, 1] element of the local Wcs affine transform
in radians.
1318 cd22 : `pandas.Series`
1319 [2, 2] element of the local Wcs affine transform
in radians.
1323 pixScale : `pandas.Series`
1324 Arcseconds per pixel at the location of the local WC
1326 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1328 def _func(self, df):
1336 """Convert a value in units pixels squared to units arcseconds squared.
1355 return f
"{self.col}_asArcseconds"
1365 def _func(self, df):
1373 """Convert a value in units pixels to units arcseconds.
1392 return f
"{self.col}_asArcsecondsSq"
1402 def _func(self, df):
1407 return df[self.
col] * pixScale * pixScale
1411 name =
'Reference Band'
1412 shortname =
'refBand'
1416 return [
"merge_measurement_i",
1417 "merge_measurement_r",
1418 "merge_measurement_z",
1419 "merge_measurement_y",
1420 "merge_measurement_g",
1421 "merge_measurement_u"]
1423 def _func(self, df: pd.DataFrame) -> pd.Series:
1424 def getFilterAliasName(row):
1426 colName = row.idxmax()
1427 return colName.replace(
'merge_measurement_',
'')
1431 columns = [col
for col
in self.
columnscolumns if col
in df.columns]
1433 return df[columns].apply(getFilterAliasName, axis=1,
1434 result_type=
'reduce').astype(
'object')
1439 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1440 LOG_AB_FLUX_SCALE = 12.56
1441 FIVE_OVER_2LOG10 = 1.085736204758129569
1445 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1451 if calib
is not None:
1465 return f
'mag_{self.col}'
1469 if np.abs(a) < np.abs(b):
1474 return np.abs(a) * np.sqrt(1. + q*q)
1480 with np.warnings.catch_warnings():
1481 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1482 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1483 return -2.5 * np.log10(dn/fluxMag0)
1486 retVal = self.
vhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1491 retVal = self.
dn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2flux(dn, fluxMag0)
1496 def _func(self, df):
1505 def _func(self, df):
1507 return pd.Series(retArr, index=df.index)
1511 def _func(self, df):
1520 def _func(self, df):
1522 return pd.Series(retArr, index=df.index)
1526 """Base class for calibrating the specified instrument flux column using
1527 the local photometric calibration.
1532 Name of the instrument flux column.
1533 instFluxErrCol : `str`
1534 Name of the assocated error columns for ``instFluxCol``.
1535 photoCalibCol : `str`
1536 Name of local calibration column.
1537 photoCalibErrCol : `str`
1538 Error associated
with ``photoCalibCol``
1548 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1563 """Convert instrument flux to nanojanskys.
1567 instFlux : `numpy.ndarray` or `pandas.Series`
1568 Array of instrument flux measurements
1569 localCalib : `numpy.ndarray`
or `pandas.Series`
1570 Array of local photometric calibration estimates.
1574 calibFlux : `numpy.ndarray`
or `pandas.Series`
1575 Array of calibrated flux measurements.
1577 return instFlux * localCalib
1580 """Convert instrument flux to nanojanskys.
1584 instFlux : `numpy.ndarray` or `pandas.Series`
1585 Array of instrument flux measurements
1586 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1587 Errors on associated ``instFlux`` values
1588 localCalib : `numpy.ndarray`
or `pandas.Series`
1589 Array of local photometric calibration estimates.
1590 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1591 Errors on associated ``localCalib`` values
1595 calibFluxErr : `numpy.ndarray`
or `pandas.Series`
1596 Errors on calibrated flux measurements.
1598 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1601 """Convert instrument flux to nanojanskys.
1605 instFlux : `numpy.ndarray` or `pandas.Series`
1606 Array of instrument flux measurements
1607 localCalib : `numpy.ndarray`
or `pandas.Series`
1608 Array of local photometric calibration estimates.
1612 calibMag : `numpy.ndarray`
or `pandas.Series`
1613 Array of calibrated AB magnitudes.
1618 """Convert instrument flux err to nanojanskys.
1622 instFlux : `numpy.ndarray` or `pandas.Series`
1623 Array of instrument flux measurements
1624 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1625 Errors on associated ``instFlux`` values
1626 localCalib : `numpy.ndarray`
or `pandas.Series`
1627 Array of local photometric calibration estimates.
1628 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1629 Errors on associated ``localCalib`` values
1633 calibMagErr: `numpy.ndarray`
or `pandas.Series`
1634 Error on calibrated AB magnitudes.
1641 """Compute calibrated fluxes using the local calibration value.
1657 return f
'flux_{self.instFluxCol}'
1659 def _func(self, df):
1664 """Compute calibrated flux errors using the local calibration value.
1681 return f
'fluxErr_{self.instFluxCol}'
1683 def _func(self, df):
1689 """Compute calibrated AB magnitudes using the local calibration value.
1705 return f
'mag_{self.instFluxCol}'
1707 def _func(self, df):
1713 """Compute calibrated AB magnitude errors using the local calibration value.
1730 return f
'magErr_{self.instFluxCol}'
1732 def _func(self, df):
1740 """Compute absolute mean of dipole fluxes.
1749 LocalDipoleMeanFluxErr
1751 LocalDipoleDiffFluxErr
1781 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1783 def _func(self, df):
1789 """Compute the error on the absolute mean of dipole fluxes.
1798 LocalDipoleMeanFluxErr
1800 LocalDipoleDiffFluxErr
1814 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1816 def _func(self, df):
1825 """Compute the absolute difference of dipole fluxes.
1827 Value is (abs(pos) - abs(neg))
1836 LocalDipoleMeanFluxErr
1838 LocalDipoleDiffFluxErr
1849 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1851 def _func(self, df):
1857 """Compute the error on the absolute difference of dipole fluxes.
1866 LocalDipoleMeanFluxErr
1868 LocalDipoleDiffFluxErr
1882 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1884 def _func(self, df):
1893 """Base class for returning the ratio of 2 columns.
1895 Can be used to compute a Signal to Noise ratio for any input flux.
1900 Name of the column to use at the numerator
in the ratio
1902 Name of the column to use
as the denominator
in the ratio.
1918 return f
'ratio_{self.numerator}_{self.denominator}'
1920 def _func(self, df):
1921 with np.warnings.catch_warnings():
1922 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1923 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1928 """Compute E(B-V) from dustmaps.sfd
1930 _defaultDataset = 'ref'
1936 from dustmaps.sfd
import SFDQuery
1937 self.
_columns = [
'coord_ra',
'coord_dec']
1941 def _func(self, df):
1942 coords = SkyCoord(df[
'coord_ra']*u.rad, df[
'coord_dec']*u.rad)
1943 ebv = self.
sfd(coords)
1946 return pd.Series(ebv, index=df.index).astype(
'float64')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr, log)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)