22__all__ = [
"Functor",
"CompositeFunctor",
"CustomFunctor",
"Column",
"Index",
23 "IDColumn",
"FootprintNPix",
"CoordColumn",
"RAColumn",
"DecColumn",
24 "HtmIndex20",
"Mag",
"MagErr",
"NanoMaggie",
"MagDiff",
"Color",
25 "Labeller",
"StarGalaxyLabeller",
"NumStarLabeller",
"DeconvolvedMoments",
26 "SdssTraceSize",
"PsfSdssTraceSizeDiff",
"HsmTraceSize",
"PsfHsmTraceSizeDiff",
27 "HsmFwhm",
"E1",
"E2",
"RadiusFromQuadrupole",
"LocalWcs",
"ComputePixelScale",
28 "ConvertPixelToArcseconds",
"ConvertPixelSqToArcsecondsSq",
"ReferenceBand",
29 "Photometry",
"NanoJansky",
"NanoJanskyErr",
"Magnitude",
"MagnitudeErr",
30 "LocalPhotometry",
"LocalNanojansky",
"LocalNanojanskyErr",
31 "LocalMagnitude",
"LocalMagnitudeErr",
"LocalDipoleMeanFlux",
32 "LocalDipoleMeanFluxErr",
"LocalDipoleDiffFlux",
"LocalDipoleDiffFluxErr",
37from itertools
import product
43import astropy.units
as u
44from astropy.coordinates
import SkyCoord
46from lsst.utils
import doImport
47from lsst.utils.introspection
import get_full_type_name
48from lsst.daf.butler
import DeferredDatasetHandle
55 typeKey='functor', name=None):
56 """Initialize an object defined in a dictionary
58 The object needs to be importable as
59 f
'{basePath}.{initDict[typeKey]}'
60 The positional
and keyword arguments (
if any) are contained
in
61 "args" and "kwargs" entries
in the dictionary, respectively.
62 This
is used
in `functors.CompositeFunctor.from_yaml` to initialize
63 a composite functor
from a specification
in a YAML file.
68 Dictionary describing object
's initialization. Must contain
69 an entry keyed by ``typeKey`` that is the name of the object,
70 relative to ``basePath``.
72 Path relative to module
in which ``initDict[typeKey]``
is defined.
74 Key of ``initDict`` that
is the name of the object
75 (relative to `basePath`).
77 initDict = initDict.copy()
79 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
81 if 'args' in initDict:
82 args = initDict.pop(
'args')
83 if isinstance(args, str):
86 element = pythonType(*args, **initDict)
87 except Exception
as e:
88 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
89 raise type(e)(message, e.args)
94 """Define and execute a calculation on a DataFrame or Handle holding a DataFrame.
96 The `__call__` method accepts either a `DataFrame` object or a
97 `DeferredDatasetHandle`
or `InMemoryDatasetHandle`,
and returns the
98 result of the calculation
as a single column. Each functor defines what
99 columns are needed
for the calculation,
and only these columns are read
100 from the dataset handle.
102 The action of `__call__` consists of two steps: first, loading the
103 necessary columns
from disk into memory
as a `pandas.DataFrame` object;
104 and second, performing the computation on this dataframe
and returning the
108 To define a new `Functor`, a subclass must define a `_func` method,
109 that takes a `pandas.DataFrame`
and returns result
in a `pandas.Series`.
110 In addition, it must define the following attributes
112 * `_columns`: The columns necessary to perform the calculation
113 * `name`: A name appropriate
for a figure axis label
114 * `shortname`: A name appropriate
for use
as a dictionary key
116 On initialization, a `Functor` should declare what band (`filt` kwarg)
117 and dataset (e.g. `
'ref'`, `
'meas'`, `
'forced_src'`) it
is intended to be
118 applied to. This enables the `_get_data` method to extract the proper
119 columns
from the underlying data. If
not specified, the dataset will fall back
120 on the `_defaultDataset`attribute. If band
is not specified
and `dataset`
121 is anything other than `
'ref'`, then an error will be raised when trying to
122 perform the calculation.
124 Originally, `Functor` was set up to expect
125 datasets formatted like the `deepCoadd_obj` dataset; that
is, a
126 dataframe
with a multi-level column index,
with the levels of the
127 column index being `band`, `dataset`,
and `column`.
128 It has since been generalized to apply to dataframes without mutli-level
129 indices
and multi-level indices
with just `dataset`
and `column` levels.
130 In addition, the `_get_data` method that reads
131 the columns
from the underlying data will
return a dataframe
with column
132 index levels defined by the `_dfLevels` attribute; by default, this
is
135 The `_dfLevels` attributes should generally
not need to
136 be changed, unless `_func` needs columns
from multiple filters
or datasets
137 to do the calculation.
139 which `_dfLevels = (
'band',
'column')`,
and `_func` expects the dataframe
140 it gets to have those levels
in the column index.
145 Filter upon which to do the calculation
148 Dataset upon which to do the calculation
149 (e.g.,
'ref',
'meas',
'forced_src').
152 _defaultDataset = 'ref'
153 _dfLevels = (
'column',)
154 _defaultNoDup =
False
156 def __init__(self, filt=None, dataset=None, noDup=None):
160 self.
log = logging.getLogger(type(self).__name__)
164 if self.
_noDup is not None:
171 """Columns required to perform calculation
173 if not hasattr(self,
'_columns'):
174 raise NotImplementedError(
'Must define columns property or _columns attribute')
177 def _get_data_columnLevels(self, data, columnIndex=None):
178 """Gets the names of the column index levels
180 This should only be called in the context of a multilevel table.
185 The data to be read, can be a `DeferredDatasetHandle`
or
186 `InMemoryDatasetHandle`.
187 columnnIndex (optional): pandas `Index` object
188 If
not passed, then it
is read
from the `DeferredDatasetHandle`
189 for `InMemoryDatasetHandle`.
191 if columnIndex
is None:
192 columnIndex = data.get(component=
"columns")
193 return columnIndex.names
195 def _get_data_columnLevelNames(self, data, columnIndex=None):
196 """Gets the content of each of the column levels for a multilevel table.
198 if columnIndex
is None:
199 columnIndex = data.get(component=
"columns")
201 columnLevels = columnIndex.names
203 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
204 for i, level
in enumerate(columnLevels)
206 return columnLevelNames
208 def _colsFromDict(self, colDict, columnIndex=None):
209 """Converts dictionary column specficiation to a list of columns
214 for i, lev
in enumerate(columnLevels):
216 if isinstance(colDict[lev], str):
217 new_colDict[lev] = [colDict[lev]]
219 new_colDict[lev] = colDict[lev]
221 new_colDict[lev] = columnIndex.levels[i]
223 levelCols = [new_colDict[lev]
for lev
in columnLevels]
224 cols = list(product(*levelCols))
225 colsAvailable = [col
for col
in cols
if col
in columnIndex]
229 """Returns columns needed by functor from multilevel dataset
231 To access tables with multilevel column structure, the `DeferredDatasetHandle`
232 or `InMemoryDatasetHandle` need to be passed either a list of tuples
or a
238 The data
as either `DeferredDatasetHandle`,
or `InMemoryDatasetHandle`.
239 columnIndex (optional): pandas `Index` object
240 either passed
or read
in from `DeferredDatasetHandle`.
241 `returnTuple` : `bool`
242 If true, then
return a list of tuples rather than the column dictionary
243 specification. This
is set to `
True` by `CompositeFunctor`
in order to be able to
244 combine columns
from the various component functors.
247 if not isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
248 raise RuntimeError(f
"Unexpected data type. Got {get_full_type_name(data)}.")
250 if columnIndex
is None:
251 columnIndex = data.get(component=
"columns")
256 columnDict = {
'column': self.
columns,
258 if self.
filt is None:
260 if "band" in columnLevels:
262 columnDict[
"band"] = columnLevelNames[
"band"][0]
264 raise ValueError(f
"'filt' not set for functor {self.name}"
265 f
"(dataset {self.dataset}) "
267 "contains multiple filters in column index. "
268 "Set 'filt' or set 'dataset' to 'ref'.")
270 columnDict[
'band'] = self.
filt
273 return self.
_colsFromDict(columnDict, columnIndex=columnIndex)
277 def _func(self, df, dropna=True):
278 raise NotImplementedError(
'Must define calculation on dataframe')
280 def _get_columnIndex(self, data):
281 """Return columnIndex
284 if isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
285 return data.get(component=
"columns")
289 def _get_data(self, data):
290 """Retrieve dataframe necessary for calculation.
292 The data argument can be a `DataFrame`, a `DeferredDatasetHandle`, or an
293 `InMemoryDatasetHandle`.
295 Returns dataframe upon which `self.
_func` can act.
299 if isinstance(data, pd.DataFrame):
300 _data = InMemoryDatasetHandle(data, storageClass=
"DataFrame")
301 elif isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
304 raise RuntimeError(f
"Unexpected type provided for data. Got {get_full_type_name(data)}.")
308 is_multiLevel = isinstance(columnIndex, pd.MultiIndex)
317 df = _data.get(parameters={
"columns": columns})
325 def _setLevels(self, df):
326 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels]
327 df.columns = df.columns.droplevel(levelsToDrop)
330 def _dropna(self, vals):
336 vals = self.
_func(df)
337 except Exception
as e:
338 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
346 """Computes difference between functor called on two different DataFrame/Handle objects
348 return self(data1, **kwargs) - self(data2, **kwargs)
351 return pd.Series(np.full(len(df), np.nan), index=df.index)
355 """Full name of functor (suitable for figure labels)
357 return NotImplementedError
361 """Short name of functor (suitable for column name/dict key)
367 """Perform multiple calculations at once on a catalog.
369 The role of a `CompositeFunctor` is to group together computations
from
370 multiple functors. Instead of returning `pandas.Series` a
371 `CompositeFunctor` returns a `pandas.Dataframe`,
with the column names
372 being the keys of `funcDict`.
374 The `columns` attribute of a `CompositeFunctor`
is the union of all columns
375 in all the component functors.
377 A `CompositeFunctor` does
not use a `_func` method itself; rather,
378 when a `CompositeFunctor`
is called, all its columns are loaded
379 at once,
and the resulting dataframe
is passed to the `_func` method of each component
380 functor. This has the advantage of only doing I/O (reading
from parquet file) once,
381 and works because each individual `_func` method of each component functor does
not
382 care
if there are *extra* columns
in the dataframe being passed; only that it must contain
383 *at least* the `columns` it expects.
385 An important
and useful
class method is `from_yaml`, which takes
as argument the path to a YAML
386 file specifying a collection of functors.
390 funcs : `dict`
or `list`
391 Dictionary
or list of functors. If a list, then it will be converted
392 into a dictonary according to the `.shortname` attribute of each functor.
396 name =
"CompositeFunctor"
400 if type(funcs) == dict:
403 self.
funcDict = {f.shortname: f
for f
in funcs}
421 if isinstance(new, dict):
423 elif isinstance(new, CompositeFunctor):
426 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
434 return list(set([x
for y
in [f.columns
for f
in self.
funcDict.values()]
for x
in y]))
443 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDict.values()
451 """Apply the functor to the data table
456 The data represented as `lsst.daf.butler.DeferredDatasetHandle`,
457 `lsst.pipe.base.InMemoryDatasetHandle`,
458 or `pandas.DataFrame`.
459 The table
or a pointer to a table on disk
from which columns can
462 if isinstance(data, pd.DataFrame):
463 _data = InMemoryDatasetHandle(data, storageClass=
"DataFrame")
464 elif isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
467 raise RuntimeError(f
"Unexpected type provided for data. Got {get_full_type_name(data)}.")
471 if isinstance(columnIndex, pd.MultiIndex):
473 df = _data.get(parameters={
"columns": columns})
478 subdf = f._setLevels(
479 df[f.multilevelColumns(_data, returnTuple=
True, columnIndex=columnIndex)]
481 valDict[k] = f._func(subdf)
482 except Exception
as e:
484 "Exception in %s (funcs: %s) call: %s",
490 valDict[k] = f.fail(subdf)
497 valDict = {k: f._func(df)
for k, f
in self.
funcDict.items()}
500 for name, colVal
in valDict.items():
501 if len(colVal.shape) != 1:
502 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
503 "It is shaped %s and type %s." % (name, colVal.shape, type(colVal)))
506 valDf = pd.concat(valDict, axis=1)
508 print([(k, type(v))
for k, v
in valDict.items()])
511 if kwargs.get(
'dropna',
False):
512 valDf = valDf.dropna(how=
'any')
518 if renameRules
is None:
520 for old, new
in renameRules:
521 if col.startswith(old):
522 col = col.replace(old, new)
528 filename = os.path.expandvars(filename)
529 with open(filename)
as f:
530 translationDefinition = yaml.safe_load(f)
532 return cls.
from_yaml(translationDefinition, **kwargs)
537 for func, val
in translationDefinition[
'funcs'].items():
540 if 'flag_rename_rules' in translationDefinition:
541 renameRules = translationDefinition[
'flag_rename_rules']
545 if 'calexpFlags' in translationDefinition:
546 for flag
in translationDefinition[
'calexpFlags']:
547 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
549 if 'refFlags' in translationDefinition:
550 for flag
in translationDefinition[
'refFlags']:
553 if 'forcedFlags' in translationDefinition:
554 for flag
in translationDefinition[
'forcedFlags']:
555 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
557 if 'flags' in translationDefinition:
558 for flag
in translationDefinition[
'flags']:
561 return cls(funcs, **kwargs)
565 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
567 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
571 df : pandas.DataFrame
572 Dataframe on which to evaluate expression.
578 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
579 val = df.eval(expr_new)
580 except Exception
as e:
581 log.error(
"Exception in mag_aware_eval: %s: %s", type(e).__name__, e)
582 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
583 val = df.eval(expr_new)
588 """Arbitrary computation on a catalog
590 Column names (and thus the columns to be loaded
from catalog) are found
591 by finding all words
and trying to ignore all
"math-y" words.
596 Expression to evaluate, to be parsed
and executed by `mag_aware_eval`.
598 _ignore_words = ('mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
610 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
expr)
612 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
expr)
if c
not in self.
_ignore_words]
615 if not re.search(
'_instFlux$', c):
616 cols.append(f
'{c}_instFlux')
621 return list(set([c
for c
in cols
if c
not in not_a_col]))
628 """Get column with specified name
648 """Return the value of the index for each object
651 columns = ['coord_ra']
652 _defaultDataset =
'ref'
656 return pd.Series(df.index, index=df.index)
661 _allow_difference =
False
665 return pd.Series(df.index, index=df.index)
669 col =
'base_Footprint_nPix'
673 """Base class for coordinate column, in degrees
682 output = df[self.
col] * 180 / np.pi
if self.
_radians else df[self.
col]
687 """Right Ascension, in degrees
693 super().
__init__(
'coord_ra', **kwargs)
696 return super().
__call__(catalog, **kwargs)
700 """Declination, in degrees
706 super().
__init__(
'coord_dec', **kwargs)
709 return super().
__call__(catalog, **kwargs)
713 """Compute the level 20 HtmIndex for the catalog.
717 This functor was implemented to satisfy requirements of old APDB interface
718 which required ``pixelId`` column in DiaObject
with HTM20 index. APDB
719 interface had migrated to
not need that information, but we keep this
720 class in case it may be useful for something else.
735 def computePixel(row):
744 return self.
pixelator.index(sphPoint.getVector())
746 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
750 if not col.endswith(
'_instFlux'):
756 if not col.endswith(
'_instFluxErr'):
757 col +=
'_instFluxErr'
762 """Compute calibrated magnitude
764 Takes a `calib` argument, which returns the flux at mag=0
765 as `calib.getFluxMag0()`. If
not provided, then the default
766 `fluxMag0`
is 63095734448.0194, which
is default
for HSC.
767 This default should be removed
in DM-21955
769 This calculation hides warnings about invalid values
and dividing by zero.
771 As
for all functors, a `dataset`
and `filt` kwarg should be provided upon
772 initialization. Unlike the default `Functor`, however, the default dataset
773 for a `Mag`
is `
'meas'`, rather than `
'ref'`.
778 Name of flux column
from which to compute magnitude. Can be parseable
779 by `lsst.pipe.tasks.functors.fluxName` function---that
is, you can
pass
780 `
'modelfit_CModel'` instead of `
'modelfit_CModel_instFlux'`)
and it will
782 calib : `lsst.afw.image.calib.Calib` (optional)
783 Object that knows zero point.
785 _defaultDataset = 'meas'
790 if calib
is not None:
803 with np.warnings.catch_warnings():
804 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
805 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
810 return f
'mag_{self.col}'
814 """Compute calibrated magnitude uncertainty
821 calib : `lsst.afw.image.calib.Calib` (optional)
822 Object that knows zero point.
827 if self.
calib is not None:
834 return [self.
col, self.
col +
'Err']
837 with np.warnings.catch_warnings():
838 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
839 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
841 x = df[fluxErrCol] / df[fluxCol]
843 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
848 return super().name +
'_err'
860 _defaultDataset =
'meas'
862 """Functor to calculate magnitude difference"""
874 with np.warnings.catch_warnings():
875 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
876 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
877 return -2.5*np.log10(df[self.
col1]/df[self.
col2])
881 return f
'(mag_{self.col1} - mag_{self.col2})'
885 return f
'magDiff_{self.col1}_{self.col2}'
889 """Compute the color between two filters
891 Computes color by initializing two different `Mag`
892 functors based on the `col` and filters provided,
and
893 then returning the difference.
895 This
is enabled by the `_func` expecting a dataframe
with a
896 multilevel column index,
with both `
'band'`
and `
'column'`,
897 instead of just `
'column'`, which
is the `Functor` default.
898 This
is controlled by the `_dfLevels` attribute.
900 Also of note, the default dataset
for `Color`
is `forced_src
'`,
901 whereas for `Mag` it
is `
'meas'`.
906 Name of flux column
from which to compute; same
as would be passed to
910 Filters
from which to compute magnitude difference.
911 Color computed
is `
Mag(filt2) -
Mag(filt1)`.
913 _defaultDataset = 'forced_src'
914 _dfLevels = (
'band',
'column')
920 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
938 mag2 = self.mag2._func(df[self.filt2])
939 mag1 = self.mag1._func(df[self.filt1])
944 return [self.
mag1.col, self.
mag2.col]
951 return f
'{self.filt2} - {self.filt1} ({self.col})'
955 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
959 """Main function of this subclass is to override the dropna=True
962 _allow_difference =
False
967 return super().
__call__(parq, dropna=
False, **kwargs)
971 _columns = [
"base_ClassificationExtendedness_value"]
972 _column =
"base_ClassificationExtendedness_value"
977 test = (x < 0.5).astype(int)
978 test = test.mask(mask, 2)
983 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
984 index=x.index, name=
'label')
986 label = label.astype(str)
991 _columns = [
'numStarFlags']
992 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
998 n = len(x.unique()) - 1
1000 labels = [
'noStar',
'maybe',
'star']
1001 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
1002 index=x.index, name=
'label')
1005 label = label.astype(str)
1011 name =
'Deconvolved Moments'
1012 shortname =
'deconvolvedMoments'
1013 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1014 "ext_shapeHSM_HsmSourceMoments_yy",
1015 "base_SdssShape_xx",
"base_SdssShape_yy",
1016 "ext_shapeHSM_HsmPsfMoments_xx",
1017 "ext_shapeHSM_HsmPsfMoments_yy")
1019 def _func(self, df):
1020 """Calculate deconvolved moments"""
1021 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1022 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1024 hsm = np.ones(len(df))*np.nan
1025 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1026 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1027 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1032 raise RuntimeError(
'No psf shape parameter found in catalog')
1034 return hsm.where(np.isfinite(hsm), sdss) - psf
1038 """Functor to calculate SDSS trace radius size for sources"""
1039 name =
"SDSS Trace Size"
1040 shortname =
'sdssTrace'
1041 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1043 def _func(self, df):
1044 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1049 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1050 name =
"PSF - SDSS Trace Size"
1051 shortname =
'psf_sdssTrace'
1052 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1053 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1055 def _func(self, df):
1056 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1057 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1058 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1063 """Functor to calculate HSM trace radius size for sources"""
1064 name =
'HSM Trace Size'
1065 shortname =
'hsmTrace'
1066 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1067 "ext_shapeHSM_HsmSourceMoments_yy")
1069 def _func(self, df):
1070 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1071 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1076 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1077 name =
'PSF - HSM Trace Size'
1078 shortname =
'psf_HsmTrace'
1079 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1080 "ext_shapeHSM_HsmSourceMoments_yy",
1081 "ext_shapeHSM_HsmPsfMoments_xx",
1082 "ext_shapeHSM_HsmPsfMoments_yy")
1084 def _func(self, df):
1085 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1086 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1087 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1088 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1089 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1094 name =
'HSM Psf FWHM'
1095 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1098 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1100 def _func(self, df):
1102 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1106 name =
"Distortion Ellipticity (e1)"
1107 shortname =
"Distortion"
1120 def _func(self, df):
1125 name =
"Ellipticity e2"
1137 def _func(self, df):
1153 def _func(self, df):
1158 """Computations using the stored localWcs.
1160 name = "LocalWcsOperations"
1175 """Compute the distance on the sphere from x2, y1 to x1, y1.
1183 cd11 : `pandas.Series`
1184 [1, 1] element of the local Wcs affine transform.
1185 cd11 : `pandas.Series`
1186 [1, 1] element of the local Wcs affine transform.
1187 cd12 : `pandas.Series`
1188 [1, 2] element of the local Wcs affine transform.
1189 cd21 : `pandas.Series`
1190 [2, 1] element of the local Wcs affine transform.
1191 cd22 : `pandas.Series`
1192 [2, 2] element of the local Wcs affine transform.
1197 RA and dec conversion of x
and y given the local Wcs. Returned
1198 units are
in radians.
1201 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1204 """Compute the local pixel scale conversion.
1208 ra1 : `pandas.Series`
1209 Ra of the first coordinate in radians.
1210 dec1 : `pandas.Series`
1211 Dec of the first coordinate
in radians.
1212 ra2 : `pandas.Series`
1213 Ra of the second coordinate
in radians.
1214 dec2 : `pandas.Series`
1215 Dec of the second coordinate
in radians.
1219 dist : `pandas.Series`
1220 Distance on the sphere
in radians.
1222 deltaDec = dec2 - dec1
1224 return 2 * np.arcsin(
1226 np.sin(deltaDec / 2) ** 2
1227 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1230 """Compute the distance on the sphere from x2, y1 to x1, y1.
1234 x1 : `pandas.Series`
1236 y1 : `pandas.Series`
1238 x2 : `pandas.Series`
1240 y2 : `pandas.Series`
1242 cd11 : `pandas.Series`
1243 [1, 1] element of the local Wcs affine transform.
1244 cd11 : `pandas.Series`
1245 [1, 1] element of the local Wcs affine transform.
1246 cd12 : `pandas.Series`
1247 [1, 2] element of the local Wcs affine transform.
1248 cd21 : `pandas.Series`
1249 [2, 1] element of the local Wcs affine transform.
1250 cd22 : `pandas.Series`
1251 [2, 2] element of the local Wcs affine transform.
1255 Distance : `pandas.Series`
1256 Arcseconds per pixel at the location of the local WC
1265 """Compute the local pixel scale from the stored CDMatrix.
1277 """Compute the local pixel to scale conversion in arcseconds.
1281 cd11 : `pandas.Series`
1282 [1, 1] element of the local Wcs affine transform in radians.
1283 cd11 : `pandas.Series`
1284 [1, 1] element of the local Wcs affine transform
in radians.
1285 cd12 : `pandas.Series`
1286 [1, 2] element of the local Wcs affine transform
in radians.
1287 cd21 : `pandas.Series`
1288 [2, 1] element of the local Wcs affine transform
in radians.
1289 cd22 : `pandas.Series`
1290 [2, 2] element of the local Wcs affine transform
in radians.
1294 pixScale : `pandas.Series`
1295 Arcseconds per pixel at the location of the local WC
1297 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1299 def _func(self, df):
1307 """Convert a value in units pixels squared to units arcseconds squared.
1326 return f
"{self.col}_asArcseconds"
1336 def _func(self, df):
1344 """Convert a value in units pixels to units arcseconds.
1363 return f
"{self.col}_asArcsecondsSq"
1373 def _func(self, df):
1378 return df[self.
col] * pixScale * pixScale
1382 name =
'Reference Band'
1383 shortname =
'refBand'
1387 return [
"merge_measurement_i",
1388 "merge_measurement_r",
1389 "merge_measurement_z",
1390 "merge_measurement_y",
1391 "merge_measurement_g",
1392 "merge_measurement_u"]
1394 def _func(self, df: pd.DataFrame) -> pd.Series:
1395 def getFilterAliasName(row):
1397 colName = row.idxmax()
1398 return colName.replace(
'merge_measurement_',
'')
1402 columns = [col
for col
in self.
columnscolumns if col
in df.columns]
1404 return df[columns].apply(getFilterAliasName, axis=1,
1405 result_type=
'reduce').astype(
'object')
1410 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1411 LOG_AB_FLUX_SCALE = 12.56
1412 FIVE_OVER_2LOG10 = 1.085736204758129569
1416 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1422 if calib
is not None:
1436 return f
'mag_{self.col}'
1440 if np.abs(a) < np.abs(b):
1445 return np.abs(a) * np.sqrt(1. + q*q)
1451 with np.warnings.catch_warnings():
1452 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1453 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1454 return -2.5 * np.log10(dn/fluxMag0)
1457 retVal = self.
vhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1462 retVal = self.
dn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2flux(dn, fluxMag0)
1467 def _func(self, df):
1476 def _func(self, df):
1478 return pd.Series(retArr, index=df.index)
1482 def _func(self, df):
1491 def _func(self, df):
1493 return pd.Series(retArr, index=df.index)
1497 """Base class for calibrating the specified instrument flux column using
1498 the local photometric calibration.
1503 Name of the instrument flux column.
1504 instFluxErrCol : `str`
1505 Name of the assocated error columns for ``instFluxCol``.
1506 photoCalibCol : `str`
1507 Name of local calibration column.
1508 photoCalibErrCol : `str`
1509 Error associated
with ``photoCalibCol``
1519 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1534 """Convert instrument flux to nanojanskys.
1538 instFlux : `numpy.ndarray` or `pandas.Series`
1539 Array of instrument flux measurements
1540 localCalib : `numpy.ndarray`
or `pandas.Series`
1541 Array of local photometric calibration estimates.
1545 calibFlux : `numpy.ndarray`
or `pandas.Series`
1546 Array of calibrated flux measurements.
1548 return instFlux * localCalib
1551 """Convert instrument flux to nanojanskys.
1555 instFlux : `numpy.ndarray` or `pandas.Series`
1556 Array of instrument flux measurements
1557 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1558 Errors on associated ``instFlux`` values
1559 localCalib : `numpy.ndarray`
or `pandas.Series`
1560 Array of local photometric calibration estimates.
1561 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1562 Errors on associated ``localCalib`` values
1566 calibFluxErr : `numpy.ndarray`
or `pandas.Series`
1567 Errors on calibrated flux measurements.
1569 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1572 """Convert instrument flux to nanojanskys.
1576 instFlux : `numpy.ndarray` or `pandas.Series`
1577 Array of instrument flux measurements
1578 localCalib : `numpy.ndarray`
or `pandas.Series`
1579 Array of local photometric calibration estimates.
1583 calibMag : `numpy.ndarray`
or `pandas.Series`
1584 Array of calibrated AB magnitudes.
1589 """Convert instrument flux err to nanojanskys.
1593 instFlux : `numpy.ndarray` or `pandas.Series`
1594 Array of instrument flux measurements
1595 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1596 Errors on associated ``instFlux`` values
1597 localCalib : `numpy.ndarray`
or `pandas.Series`
1598 Array of local photometric calibration estimates.
1599 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1600 Errors on associated ``localCalib`` values
1604 calibMagErr: `numpy.ndarray`
or `pandas.Series`
1605 Error on calibrated AB magnitudes.
1612 """Compute calibrated fluxes using the local calibration value.
1628 return f
'flux_{self.instFluxCol}'
1630 def _func(self, df):
1635 """Compute calibrated flux errors using the local calibration value.
1652 return f
'fluxErr_{self.instFluxCol}'
1654 def _func(self, df):
1660 """Compute calibrated AB magnitudes using the local calibration value.
1676 return f
'mag_{self.instFluxCol}'
1678 def _func(self, df):
1684 """Compute calibrated AB magnitude errors using the local calibration value.
1701 return f
'magErr_{self.instFluxCol}'
1703 def _func(self, df):
1711 """Compute absolute mean of dipole fluxes.
1720 LocalDipoleMeanFluxErr
1722 LocalDipoleDiffFluxErr
1752 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1754 def _func(self, df):
1760 """Compute the error on the absolute mean of dipole fluxes.
1769 LocalDipoleMeanFluxErr
1771 LocalDipoleDiffFluxErr
1785 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1787 def _func(self, df):
1796 """Compute the absolute difference of dipole fluxes.
1798 Value is (abs(pos) - abs(neg))
1807 LocalDipoleMeanFluxErr
1809 LocalDipoleDiffFluxErr
1820 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1822 def _func(self, df):
1828 """Compute the error on the absolute difference of dipole fluxes.
1837 LocalDipoleMeanFluxErr
1839 LocalDipoleDiffFluxErr
1853 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1855 def _func(self, df):
1864 """Base class for returning the ratio of 2 columns.
1866 Can be used to compute a Signal to Noise ratio for any input flux.
1871 Name of the column to use at the numerator
in the ratio
1873 Name of the column to use
as the denominator
in the ratio.
1889 return f
'ratio_{self.numerator}_{self.denominator}'
1891 def _func(self, df):
1892 with np.warnings.catch_warnings():
1893 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1894 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1899 """Compute E(B-V) from dustmaps.sfd
1901 _defaultDataset = 'ref'
1907 from dustmaps.sfd
import SFDQuery
1908 self.
_columns = [
'coord_ra',
'coord_dec']
1912 def _func(self, df):
1913 coords = SkyCoord(df[
'coord_ra'].values * u.rad, df[
'coord_dec'].values * u.rad)
1914 ebv = self.
sfd(coords)
1917 return pd.Series(ebv, index=df.index).astype(
'float64')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def getSkySeparationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def computeSkySeparation(self, ra1, dec1, ra2, dec2)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr, log)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)