22__all__ = [
"Functor",
"CompositeFunctor",
"CustomFunctor",
"Column",
"Index",
23 "IDColumn",
"FootprintNPix",
"CoordColumn",
"RAColumn",
"DecColumn",
24 "HtmIndex20",
"Mag",
"MagErr",
"NanoMaggie",
"MagDiff",
"Color",
25 "Labeller",
"StarGalaxyLabeller",
"NumStarLabeller",
"DeconvolvedMoments",
26 "SdssTraceSize",
"PsfSdssTraceSizeDiff",
"HsmTraceSize",
"PsfHsmTraceSizeDiff",
27 "HsmFwhm",
"E1",
"E2",
"RadiusFromQuadrupole",
"LocalWcs",
"ComputePixelScale",
28 "ConvertPixelToArcseconds",
"ConvertPixelSqToArcsecondsSq",
"ReferenceBand",
29 "Photometry",
"NanoJansky",
"NanoJanskyErr",
"Magnitude",
"MagnitudeErr",
30 "LocalPhotometry",
"LocalNanojansky",
"LocalNanojanskyErr",
31 "LocalMagnitude",
"LocalMagnitudeErr",
"LocalDipoleMeanFlux",
32 "LocalDipoleMeanFluxErr",
"LocalDipoleDiffFlux",
"LocalDipoleDiffFluxErr",
37from itertools
import product
43import astropy.units
as u
44from astropy.coordinates
import SkyCoord
46from lsst.utils
import doImport
47from lsst.utils.introspection
import get_full_type_name
48from lsst.daf.butler
import DeferredDatasetHandle
55 typeKey='functor', name=None):
56 """Initialize an object defined in a dictionary
58 The object needs to be importable as
59 f
'{basePath}.{initDict[typeKey]}'
60 The positional
and keyword arguments (
if any) are contained
in
61 "args" and "kwargs" entries
in the dictionary, respectively.
62 This
is used
in `functors.CompositeFunctor.from_yaml` to initialize
63 a composite functor
from a specification
in a YAML file.
68 Dictionary describing object
's initialization. Must contain
69 an entry keyed by ``typeKey`` that is the name of the object,
70 relative to ``basePath``.
72 Path relative to module
in which ``initDict[typeKey]``
is defined.
74 Key of ``initDict`` that
is the name of the object
75 (relative to `basePath`).
77 initDict = initDict.copy()
79 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
81 if 'args' in initDict:
82 args = initDict.pop(
'args')
83 if isinstance(args, str):
86 element = pythonType(*args, **initDict)
87 except Exception
as e:
88 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
89 raise type(e)(message, e.args)
94 """Define and execute a calculation on a DataFrame or Handle holding a DataFrame.
96 The `__call__` method accepts either a `DataFrame` object or a
97 `DeferredDatasetHandle`
or `InMemoryDatasetHandle`,
and returns the
98 result of the calculation
as a single column. Each functor defines what
99 columns are needed
for the calculation,
and only these columns are read
100 from the dataset handle.
102 The action of `__call__` consists of two steps: first, loading the
103 necessary columns
from disk into memory
as a `pandas.DataFrame` object;
104 and second, performing the computation on this dataframe
and returning the
108 To define a new `Functor`, a subclass must define a `_func` method,
109 that takes a `pandas.DataFrame`
and returns result
in a `pandas.Series`.
110 In addition, it must define the following attributes
112 * `_columns`: The columns necessary to perform the calculation
113 * `name`: A name appropriate
for a figure axis label
114 * `shortname`: A name appropriate
for use
as a dictionary key
116 On initialization, a `Functor` should declare what band (`filt` kwarg)
117 and dataset (e.g. `
'ref'`, `
'meas'`, `
'forced_src'`) it
is intended to be
118 applied to. This enables the `_get_data` method to extract the proper
119 columns
from the underlying data. If
not specified, the dataset will fall back
120 on the `_defaultDataset`attribute. If band
is not specified
and `dataset`
121 is anything other than `
'ref'`, then an error will be raised when trying to
122 perform the calculation.
124 Originally, `Functor` was set up to expect
125 datasets formatted like the `deepCoadd_obj` dataset; that
is, a
126 dataframe
with a multi-level column index,
with the levels of the
127 column index being `band`, `dataset`,
and `column`.
128 It has since been generalized to apply to dataframes without mutli-level
129 indices
and multi-level indices
with just `dataset`
and `column` levels.
130 In addition, the `_get_data` method that reads
131 the columns
from the underlying data will
return a dataframe
with column
132 index levels defined by the `_dfLevels` attribute; by default, this
is
135 The `_dfLevels` attributes should generally
not need to
136 be changed, unless `_func` needs columns
from multiple filters
or datasets
137 to do the calculation.
139 which `_dfLevels = (
'band',
'column')`,
and `_func` expects the dataframe
140 it gets to have those levels
in the column index.
145 Filter upon which to do the calculation
148 Dataset upon which to do the calculation
149 (e.g.,
'ref',
'meas',
'forced_src').
152 _defaultDataset = 'ref'
153 _dfLevels = (
'column',)
154 _defaultNoDup =
False
156 def __init__(self, filt=None, dataset=None, noDup=None):
160 self.
log = logging.getLogger(type(self).__name__)
164 if self.
_noDup is not None:
171 """Columns required to perform calculation
173 if not hasattr(self,
'_columns'):
174 raise NotImplementedError(
'Must define columns property or _columns attribute')
177 def _get_data_columnLevels(self, data, columnIndex=None):
178 """Gets the names of the column index levels
180 This should only be called in the context of a multilevel table.
185 The data to be read, can be a `DeferredDatasetHandle`
or
186 `InMemoryDatasetHandle`.
187 columnnIndex (optional): pandas `Index` object
188 If
not passed, then it
is read
from the `DeferredDatasetHandle`
189 for `InMemoryDatasetHandle`.
191 if columnIndex
is None:
192 columnIndex = data.get(component=
"columns")
193 return columnIndex.names
195 def _get_data_columnLevelNames(self, data, columnIndex=None):
196 """Gets the content of each of the column levels for a multilevel table.
198 if columnIndex
is None:
199 columnIndex = data.get(component=
"columns")
201 columnLevels = columnIndex.names
203 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
204 for i, level
in enumerate(columnLevels)
206 return columnLevelNames
208 def _colsFromDict(self, colDict, columnIndex=None):
209 """Converts dictionary column specficiation to a list of columns
214 for i, lev
in enumerate(columnLevels):
216 if isinstance(colDict[lev], str):
217 new_colDict[lev] = [colDict[lev]]
219 new_colDict[lev] = colDict[lev]
221 new_colDict[lev] = columnIndex.levels[i]
223 levelCols = [new_colDict[lev]
for lev
in columnLevels]
224 cols = list(product(*levelCols))
225 colsAvailable = [col
for col
in cols
if col
in columnIndex]
229 """Returns columns needed by functor from multilevel dataset
231 To access tables with multilevel column structure, the `DeferredDatasetHandle`
232 or `InMemoryDatasetHandle` need to be passed either a list of tuples
or a
238 The data
as either `DeferredDatasetHandle`,
or `InMemoryDatasetHandle`.
239 columnIndex (optional): pandas `Index` object
240 either passed
or read
in from `DeferredDatasetHandle`.
241 `returnTuple` : `bool`
242 If true, then
return a list of tuples rather than the column dictionary
243 specification. This
is set to `
True` by `CompositeFunctor`
in order to be able to
244 combine columns
from the various component functors.
247 if not isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
248 raise RuntimeError(f
"Unexpected data type. Got {get_full_type_name(data)}.")
250 if columnIndex
is None:
251 columnIndex = data.get(component=
"columns")
256 columnDict = {
'column': self.
columns,
258 if self.
filt is None:
260 if "band" in columnLevels:
262 columnDict[
"band"] = columnLevelNames[
"band"][0]
264 raise ValueError(f
"'filt' not set for functor {self.name}"
265 f
"(dataset {self.dataset}) "
267 "contains multiple filters in column index. "
268 "Set 'filt' or set 'dataset' to 'ref'.")
270 columnDict[
'band'] = self.
filt
273 return self.
_colsFromDict(columnDict, columnIndex=columnIndex)
277 def _func(self, df, dropna=True):
278 raise NotImplementedError(
'Must define calculation on dataframe')
280 def _get_columnIndex(self, data):
281 """Return columnIndex
284 if isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
285 return data.get(component=
"columns")
289 def _get_data(self, data):
290 """Retrieve dataframe necessary for calculation.
292 The data argument can be a `DataFrame`, a `DeferredDatasetHandle`, or an
293 `InMemoryDatasetHandle`.
295 Returns dataframe upon which `self.
_func` can act.
299 if isinstance(data, pd.DataFrame):
300 _data = InMemoryDatasetHandle(data, storageClass=
"DataFrame")
301 elif isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
304 raise RuntimeError(f
"Unexpected type provided for data. Got {get_full_type_name(data)}.")
308 is_multiLevel = isinstance(columnIndex, pd.MultiIndex)
317 df = _data.get(parameters={
"columns": columns})
325 def _setLevels(self, df):
326 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels]
327 df.columns = df.columns.droplevel(levelsToDrop)
330 def _dropna(self, vals):
336 vals = self.
_func(df)
337 except Exception
as e:
338 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
346 """Computes difference between functor called on two different DataFrame/Handle objects
348 return self(data1, **kwargs) - self(data2, **kwargs)
351 return pd.Series(np.full(len(df), np.nan), index=df.index)
355 """Full name of functor (suitable for figure labels)
357 return NotImplementedError
361 """Short name of functor (suitable for column name/dict key)
367 """Perform multiple calculations at once on a catalog.
369 The role of a `CompositeFunctor` is to group together computations
from
370 multiple functors. Instead of returning `pandas.Series` a
371 `CompositeFunctor` returns a `pandas.Dataframe`,
with the column names
372 being the keys of `funcDict`.
374 The `columns` attribute of a `CompositeFunctor`
is the union of all columns
375 in all the component functors.
377 A `CompositeFunctor` does
not use a `_func` method itself; rather,
378 when a `CompositeFunctor`
is called, all its columns are loaded
379 at once,
and the resulting dataframe
is passed to the `_func` method of each component
380 functor. This has the advantage of only doing I/O (reading
from parquet file) once,
381 and works because each individual `_func` method of each component functor does
not
382 care
if there are *extra* columns
in the dataframe being passed; only that it must contain
383 *at least* the `columns` it expects.
385 An important
and useful
class method is `from_yaml`, which takes
as argument the path to a YAML
386 file specifying a collection of functors.
390 funcs : `dict`
or `list`
391 Dictionary
or list of functors. If a list, then it will be converted
392 into a dictonary according to the `.shortname` attribute of each functor.
399 if type(funcs) == dict:
402 self.
funcDict = {f.shortname: f
for f
in funcs}
420 if isinstance(new, dict):
422 elif isinstance(new, CompositeFunctor):
425 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
433 return list(set([x
for y
in [f.columns
for f
in self.
funcDict.values()]
for x
in y]))
442 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDict.values()
450 """Apply the functor to the data table
455 The data represented as `lsst.daf.butler.DeferredDatasetHandle`,
456 `lsst.pipe.base.InMemoryDatasetHandle`,
457 or `pandas.DataFrame`.
458 The table
or a pointer to a table on disk
from which columns can
461 if isinstance(data, pd.DataFrame):
462 _data = InMemoryDatasetHandle(data, storageClass=
"DataFrame")
463 elif isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
466 raise RuntimeError(f
"Unexpected type provided for data. Got {get_full_type_name(data)}.")
470 if isinstance(columnIndex, pd.MultiIndex):
472 df = _data.get(parameters={
"columns": columns})
477 subdf = f._setLevels(
478 df[f.multilevelColumns(_data, returnTuple=
True, columnIndex=columnIndex)]
480 valDict[k] = f._func(subdf)
481 except Exception
as e:
482 self.
log.error(
"Exception in %s call: %s: %s", self.
name, type(e).__name__, e)
484 valDict[k] = f.fail(subdf)
491 valDict = {k: f._func(df)
for k, f
in self.
funcDict.items()}
494 for name, colVal
in valDict.items():
495 if len(colVal.shape) != 1:
496 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
497 "It is shaped %s and type %s." % (name, colVal.shape, type(colVal)))
500 valDf = pd.concat(valDict, axis=1)
502 print([(k, type(v))
for k, v
in valDict.items()])
505 if kwargs.get(
'dropna',
False):
506 valDf = valDf.dropna(how=
'any')
512 if renameRules
is None:
514 for old, new
in renameRules:
515 if col.startswith(old):
516 col = col.replace(old, new)
522 filename = os.path.expandvars(filename)
523 with open(filename)
as f:
524 translationDefinition = yaml.safe_load(f)
526 return cls.
from_yaml(translationDefinition, **kwargs)
531 for func, val
in translationDefinition[
'funcs'].items():
534 if 'flag_rename_rules' in translationDefinition:
535 renameRules = translationDefinition[
'flag_rename_rules']
539 if 'calexpFlags' in translationDefinition:
540 for flag
in translationDefinition[
'calexpFlags']:
541 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
543 if 'refFlags' in translationDefinition:
544 for flag
in translationDefinition[
'refFlags']:
547 if 'forcedFlags' in translationDefinition:
548 for flag
in translationDefinition[
'forcedFlags']:
549 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
551 if 'flags' in translationDefinition:
552 for flag
in translationDefinition[
'flags']:
555 return cls(funcs, **kwargs)
559 """Evaluate an expression on a DataFrame, knowing what the 'mag' function means
561 Builds on `pandas.DataFrame.eval`, which parses and executes math on dataframes.
565 df : pandas.DataFrame
566 Dataframe on which to evaluate expression.
572 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
573 val = df.eval(expr_new)
574 except Exception
as e:
575 log.error(
"Exception in mag_aware_eval: %s: %s", type(e).__name__, e)
576 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
577 val = df.eval(expr_new)
582 """Arbitrary computation on a catalog
584 Column names (and thus the columns to be loaded
from catalog) are found
585 by finding all words
and trying to ignore all
"math-y" words.
590 Expression to evaluate, to be parsed
and executed by `mag_aware_eval`.
592 _ignore_words = ('mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
604 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
expr)
606 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
expr)
if c
not in self.
_ignore_words]
609 if not re.search(
'_instFlux$', c):
610 cols.append(f
'{c}_instFlux')
615 return list(set([c
for c
in cols
if c
not in not_a_col]))
622 """Get column with specified name
642 """Return the value of the index for each object
645 columns = ['coord_ra']
646 _defaultDataset =
'ref'
650 return pd.Series(df.index, index=df.index)
655 _allow_difference =
False
659 return pd.Series(df.index, index=df.index)
663 col =
'base_Footprint_nPix'
667 """Base class for coordinate column, in degrees
676 output = df[self.
col] * 180 / np.pi
if self.
_radians else df[self.
col]
681 """Right Ascension, in degrees
687 super().
__init__(
'coord_ra', **kwargs)
690 return super().
__call__(catalog, **kwargs)
694 """Declination, in degrees
700 super().
__init__(
'coord_dec', **kwargs)
703 return super().
__call__(catalog, **kwargs)
707 """Compute the level 20 HtmIndex for the catalog.
711 This functor was implemented to satisfy requirements of old APDB interface
712 which required ``pixelId`` column in DiaObject
with HTM20 index. APDB
713 interface had migrated to
not need that information, but we keep this
714 class in case it may be useful for something else.
729 def computePixel(row):
738 return self.
pixelator.index(sphPoint.getVector())
740 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
744 if not col.endswith(
'_instFlux'):
750 if not col.endswith(
'_instFluxErr'):
751 col +=
'_instFluxErr'
756 """Compute calibrated magnitude
758 Takes a `calib` argument, which returns the flux at mag=0
759 as `calib.getFluxMag0()`. If
not provided, then the default
760 `fluxMag0`
is 63095734448.0194, which
is default
for HSC.
761 This default should be removed
in DM-21955
763 This calculation hides warnings about invalid values
and dividing by zero.
765 As
for all functors, a `dataset`
and `filt` kwarg should be provided upon
766 initialization. Unlike the default `Functor`, however, the default dataset
767 for a `Mag`
is `
'meas'`, rather than `
'ref'`.
772 Name of flux column
from which to compute magnitude. Can be parseable
773 by `lsst.pipe.tasks.functors.fluxName` function---that
is, you can
pass
774 `
'modelfit_CModel'` instead of `
'modelfit_CModel_instFlux'`)
and it will
776 calib : `lsst.afw.image.calib.Calib` (optional)
777 Object that knows zero point.
779 _defaultDataset = 'meas'
784 if calib
is not None:
797 with np.warnings.catch_warnings():
798 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
799 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
804 return f
'mag_{self.col}'
808 """Compute calibrated magnitude uncertainty
815 calib : `lsst.afw.image.calib.Calib` (optional)
816 Object that knows zero point.
821 if self.
calib is not None:
828 return [self.
col, self.
col +
'Err']
831 with np.warnings.catch_warnings():
832 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
833 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
835 x = df[fluxErrCol] / df[fluxCol]
837 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
842 return super().name +
'_err'
854 _defaultDataset =
'meas'
856 """Functor to calculate magnitude difference"""
868 with np.warnings.catch_warnings():
869 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
870 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
871 return -2.5*np.log10(df[self.
col1]/df[self.
col2])
875 return f
'(mag_{self.col1} - mag_{self.col2})'
879 return f
'magDiff_{self.col1}_{self.col2}'
883 """Compute the color between two filters
885 Computes color by initializing two different `Mag`
886 functors based on the `col` and filters provided,
and
887 then returning the difference.
889 This
is enabled by the `_func` expecting a dataframe
with a
890 multilevel column index,
with both `
'band'`
and `
'column'`,
891 instead of just `
'column'`, which
is the `Functor` default.
892 This
is controlled by the `_dfLevels` attribute.
894 Also of note, the default dataset
for `Color`
is `forced_src
'`,
895 whereas for `Mag` it
is `
'meas'`.
900 Name of flux column
from which to compute; same
as would be passed to
904 Filters
from which to compute magnitude difference.
905 Color computed
is `
Mag(filt2) -
Mag(filt1)`.
907 _defaultDataset = 'forced_src'
908 _dfLevels = (
'band',
'column')
914 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
932 mag2 = self.mag2._func(df[self.filt2])
933 mag1 = self.mag1._func(df[self.filt1])
938 return [self.
mag1.col, self.
mag2.col]
945 return f
'{self.filt2} - {self.filt1} ({self.col})'
949 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
953 """Main function of this subclass is to override the dropna=True
956 _allow_difference =
False
961 return super().
__call__(parq, dropna=
False, **kwargs)
965 _columns = [
"base_ClassificationExtendedness_value"]
966 _column =
"base_ClassificationExtendedness_value"
971 test = (x < 0.5).astype(int)
972 test = test.mask(mask, 2)
977 label = pd.Series(pd.Categorical.from_codes(test, categories=categories),
978 index=x.index, name=
'label')
980 label = label.astype(str)
985 _columns = [
'numStarFlags']
986 labels = {
"star": 0,
"maybe": 1,
"notStar": 2}
992 n = len(x.unique()) - 1
994 labels = [
'noStar',
'maybe',
'star']
995 label = pd.Series(pd.cut(x, [-1, 0, n-1, n], labels=labels),
996 index=x.index, name=
'label')
999 label = label.astype(str)
1005 name =
'Deconvolved Moments'
1006 shortname =
'deconvolvedMoments'
1007 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1008 "ext_shapeHSM_HsmSourceMoments_yy",
1009 "base_SdssShape_xx",
"base_SdssShape_yy",
1010 "ext_shapeHSM_HsmPsfMoments_xx",
1011 "ext_shapeHSM_HsmPsfMoments_yy")
1013 def _func(self, df):
1014 """Calculate deconvolved moments"""
1015 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
1016 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
1018 hsm = np.ones(len(df))*np.nan
1019 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
1020 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
1021 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1026 raise RuntimeError(
'No psf shape parameter found in catalog')
1028 return hsm.where(np.isfinite(hsm), sdss) - psf
1032 """Functor to calculate SDSS trace radius size for sources"""
1033 name =
"SDSS Trace Size"
1034 shortname =
'sdssTrace'
1035 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1037 def _func(self, df):
1038 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1043 """Functor to calculate SDSS trace radius size difference (%) between object and psf model"""
1044 name =
"PSF - SDSS Trace Size"
1045 shortname =
'psf_sdssTrace'
1046 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1047 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1049 def _func(self, df):
1050 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1051 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1052 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1057 """Functor to calculate HSM trace radius size for sources"""
1058 name =
'HSM Trace Size'
1059 shortname =
'hsmTrace'
1060 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1061 "ext_shapeHSM_HsmSourceMoments_yy")
1063 def _func(self, df):
1064 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1065 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1070 """Functor to calculate HSM trace radius size difference (%) between object and psf model"""
1071 name =
'PSF - HSM Trace Size'
1072 shortname =
'psf_HsmTrace'
1073 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1074 "ext_shapeHSM_HsmSourceMoments_yy",
1075 "ext_shapeHSM_HsmPsfMoments_xx",
1076 "ext_shapeHSM_HsmPsfMoments_yy")
1078 def _func(self, df):
1079 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1080 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1081 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1082 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1083 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1088 name =
'HSM Psf FWHM'
1089 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1092 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1094 def _func(self, df):
1096 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1100 name =
"Distortion Ellipticity (e1)"
1101 shortname =
"Distortion"
1114 def _func(self, df):
1119 name =
"Ellipticity e2"
1131 def _func(self, df):
1147 def _func(self, df):
1152 """Computations using the stored localWcs.
1154 name = "LocalWcsOperations"
1169 """Compute the distance on the sphere from x2, y1 to x1, y1.
1177 cd11 : `pandas.Series`
1178 [1, 1] element of the local Wcs affine transform.
1179 cd11 : `pandas.Series`
1180 [1, 1] element of the local Wcs affine transform.
1181 cd12 : `pandas.Series`
1182 [1, 2] element of the local Wcs affine transform.
1183 cd21 : `pandas.Series`
1184 [2, 1] element of the local Wcs affine transform.
1185 cd22 : `pandas.Series`
1186 [2, 2] element of the local Wcs affine transform.
1191 RA and dec conversion of x
and y given the local Wcs. Returned
1192 units are
in radians.
1195 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1198 """Compute the local pixel scale conversion.
1202 ra1 : `pandas.Series`
1203 Ra of the first coordinate in radians.
1204 dec1 : `pandas.Series`
1205 Dec of the first coordinate
in radians.
1206 ra2 : `pandas.Series`
1207 Ra of the second coordinate
in radians.
1208 dec2 : `pandas.Series`
1209 Dec of the second coordinate
in radians.
1213 dist : `pandas.Series`
1214 Distance on the sphere
in radians.
1216 deltaDec = dec2 - dec1
1218 return 2 * np.arcsin(
1220 np.sin(deltaDec / 2) ** 2
1221 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1224 """Compute the distance on the sphere from x2, y1 to x1, y1.
1228 x1 : `pandas.Series`
1230 y1 : `pandas.Series`
1232 x2 : `pandas.Series`
1234 y2 : `pandas.Series`
1236 cd11 : `pandas.Series`
1237 [1, 1] element of the local Wcs affine transform.
1238 cd11 : `pandas.Series`
1239 [1, 1] element of the local Wcs affine transform.
1240 cd12 : `pandas.Series`
1241 [1, 2] element of the local Wcs affine transform.
1242 cd21 : `pandas.Series`
1243 [2, 1] element of the local Wcs affine transform.
1244 cd22 : `pandas.Series`
1245 [2, 2] element of the local Wcs affine transform.
1249 Distance : `pandas.Series`
1250 Arcseconds per pixel at the location of the local WC
1259 """Compute the local pixel scale from the stored CDMatrix.
1271 """Compute the local pixel to scale conversion in arcseconds.
1275 cd11 : `pandas.Series`
1276 [1, 1] element of the local Wcs affine transform in radians.
1277 cd11 : `pandas.Series`
1278 [1, 1] element of the local Wcs affine transform
in radians.
1279 cd12 : `pandas.Series`
1280 [1, 2] element of the local Wcs affine transform
in radians.
1281 cd21 : `pandas.Series`
1282 [2, 1] element of the local Wcs affine transform
in radians.
1283 cd22 : `pandas.Series`
1284 [2, 2] element of the local Wcs affine transform
in radians.
1288 pixScale : `pandas.Series`
1289 Arcseconds per pixel at the location of the local WC
1291 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1293 def _func(self, df):
1301 """Convert a value in units pixels squared to units arcseconds squared.
1320 return f
"{self.col}_asArcseconds"
1330 def _func(self, df):
1338 """Convert a value in units pixels to units arcseconds.
1357 return f
"{self.col}_asArcsecondsSq"
1367 def _func(self, df):
1372 return df[self.
col] * pixScale * pixScale
1376 name =
'Reference Band'
1377 shortname =
'refBand'
1381 return [
"merge_measurement_i",
1382 "merge_measurement_r",
1383 "merge_measurement_z",
1384 "merge_measurement_y",
1385 "merge_measurement_g",
1386 "merge_measurement_u"]
1388 def _func(self, df: pd.DataFrame) -> pd.Series:
1389 def getFilterAliasName(row):
1391 colName = row.idxmax()
1392 return colName.replace(
'merge_measurement_',
'')
1396 columns = [col
for col
in self.
columnscolumns if col
in df.columns]
1398 return df[columns].apply(getFilterAliasName, axis=1,
1399 result_type=
'reduce').astype(
'object')
1404 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1405 LOG_AB_FLUX_SCALE = 12.56
1406 FIVE_OVER_2LOG10 = 1.085736204758129569
1410 def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs):
1416 if calib
is not None:
1430 return f
'mag_{self.col}'
1434 if np.abs(a) < np.abs(b):
1439 return np.abs(a) * np.sqrt(1. + q*q)
1445 with np.warnings.catch_warnings():
1446 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1447 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1448 return -2.5 * np.log10(dn/fluxMag0)
1451 retVal = self.
vhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1456 retVal = self.
dn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2flux(dn, fluxMag0)
1461 def _func(self, df):
1470 def _func(self, df):
1472 return pd.Series(retArr, index=df.index)
1476 def _func(self, df):
1485 def _func(self, df):
1487 return pd.Series(retArr, index=df.index)
1491 """Base class for calibrating the specified instrument flux column using
1492 the local photometric calibration.
1497 Name of the instrument flux column.
1498 instFluxErrCol : `str`
1499 Name of the assocated error columns for ``instFluxCol``.
1500 photoCalibCol : `str`
1501 Name of local calibration column.
1502 photoCalibErrCol : `str`
1503 Error associated
with ``photoCalibCol``
1513 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1528 """Convert instrument flux to nanojanskys.
1532 instFlux : `numpy.ndarray` or `pandas.Series`
1533 Array of instrument flux measurements
1534 localCalib : `numpy.ndarray`
or `pandas.Series`
1535 Array of local photometric calibration estimates.
1539 calibFlux : `numpy.ndarray`
or `pandas.Series`
1540 Array of calibrated flux measurements.
1542 return instFlux * localCalib
1545 """Convert instrument flux to nanojanskys.
1549 instFlux : `numpy.ndarray` or `pandas.Series`
1550 Array of instrument flux measurements
1551 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1552 Errors on associated ``instFlux`` values
1553 localCalib : `numpy.ndarray`
or `pandas.Series`
1554 Array of local photometric calibration estimates.
1555 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1556 Errors on associated ``localCalib`` values
1560 calibFluxErr : `numpy.ndarray`
or `pandas.Series`
1561 Errors on calibrated flux measurements.
1563 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1566 """Convert instrument flux to nanojanskys.
1570 instFlux : `numpy.ndarray` or `pandas.Series`
1571 Array of instrument flux measurements
1572 localCalib : `numpy.ndarray`
or `pandas.Series`
1573 Array of local photometric calibration estimates.
1577 calibMag : `numpy.ndarray`
or `pandas.Series`
1578 Array of calibrated AB magnitudes.
1583 """Convert instrument flux err to nanojanskys.
1587 instFlux : `numpy.ndarray` or `pandas.Series`
1588 Array of instrument flux measurements
1589 instFluxErr : `numpy.ndarray`
or `pandas.Series`
1590 Errors on associated ``instFlux`` values
1591 localCalib : `numpy.ndarray`
or `pandas.Series`
1592 Array of local photometric calibration estimates.
1593 localCalibErr : `numpy.ndarray`
or `pandas.Series`
1594 Errors on associated ``localCalib`` values
1598 calibMagErr: `numpy.ndarray`
or `pandas.Series`
1599 Error on calibrated AB magnitudes.
1606 """Compute calibrated fluxes using the local calibration value.
1622 return f
'flux_{self.instFluxCol}'
1624 def _func(self, df):
1629 """Compute calibrated flux errors using the local calibration value.
1646 return f
'fluxErr_{self.instFluxCol}'
1648 def _func(self, df):
1654 """Compute calibrated AB magnitudes using the local calibration value.
1670 return f
'mag_{self.instFluxCol}'
1672 def _func(self, df):
1678 """Compute calibrated AB magnitude errors using the local calibration value.
1695 return f
'magErr_{self.instFluxCol}'
1697 def _func(self, df):
1705 """Compute absolute mean of dipole fluxes.
1714 LocalDipoleMeanFluxErr
1716 LocalDipoleDiffFluxErr
1746 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1748 def _func(self, df):
1754 """Compute the error on the absolute mean of dipole fluxes.
1763 LocalDipoleMeanFluxErr
1765 LocalDipoleDiffFluxErr
1779 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1781 def _func(self, df):
1790 """Compute the absolute difference of dipole fluxes.
1792 Value is (abs(pos) - abs(neg))
1801 LocalDipoleMeanFluxErr
1803 LocalDipoleDiffFluxErr
1814 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1816 def _func(self, df):
1822 """Compute the error on the absolute difference of dipole fluxes.
1831 LocalDipoleMeanFluxErr
1833 LocalDipoleDiffFluxErr
1847 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1849 def _func(self, df):
1858 """Base class for returning the ratio of 2 columns.
1860 Can be used to compute a Signal to Noise ratio for any input flux.
1865 Name of the column to use at the numerator
in the ratio
1867 Name of the column to use
as the denominator
in the ratio.
1883 return f
'ratio_{self.numerator}_{self.denominator}'
1885 def _func(self, df):
1886 with np.warnings.catch_warnings():
1887 np.warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1888 np.warnings.filterwarnings(
'ignore',
r'divide by zero')
1893 """Compute E(B-V) from dustmaps.sfd
1895 _defaultDataset = 'ref'
1901 from dustmaps.sfd
import SFDQuery
1902 self.
_columns = [
'coord_ra',
'coord_dec']
1906 def _func(self, df):
1907 coords = SkyCoord(df[
'coord_ra']*u.rad, df[
'coord_dec']*u.rad)
1908 ebv = self.
sfd(coords)
1911 return pd.Series(ebv, index=df.index).astype(
'float64')
def multilevelColumns(self, parq, **kwargs)
def __init__(self, col, filt2, filt1, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, funcs, **kwargs)
def __call__(self, data, **kwargs)
def from_file(cls, filename, **kwargs)
def from_yaml(cls, translationDefinition, **kwargs)
def renameCol(cls, col, renameRules)
def multilevelColumns(self, data, **kwargs)
def pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def __init__(self, col, **kwargs)
def __init__(self, expr, **kwargs)
def __init__(self, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, **kwargs)
def __call__(self, data, dropna=False)
def _get_data(self, data)
def _func(self, df, dropna=True)
def multilevelColumns(self, data, columnIndex=None, returnTuple=False)
def _get_data_columnLevelNames(self, data, columnIndex=None)
def difference(self, data1, data2, **kwargs)
def __init__(self, filt=None, dataset=None, noDup=None)
def _get_columnIndex(self, data)
def _colsFromDict(self, colDict, columnIndex=None)
def _get_data_columnLevels(self, data, columnIndex=None)
def __init__(self, ra, decl, **kwargs)
def __call__(self, parq, dropna=False, **kwargs)
def __init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxToNanojansky(self, instFlux, localCalib)
def instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def __init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
def instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
def instFluxToMagnitude(self, instFlux, localCalib)
def __init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
def computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
def computeSkySeperation(self, ra1, dec1, ra2, dec2)
def getSkySeperationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
def __init__(self, col1, col2, **kwargs)
def __init__(self, *args, **kwargs)
def __init__(self, col, calib=None, **kwargs)
def dn2mag(self, dn, fluxMag0)
def dn2flux(self, dn, fluxMag0)
def dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
def __init__(self, colFlux, colFluxErr=None, calib=None, **kwargs)
def __call__(self, catalog, **kwargs)
def __init__(self, **kwargs)
def __init__(self, colXX, colXY, colYY, **kwargs)
def __init__(self, numerator, denominator, **kwargs)
def mag_aware_eval(df, expr, log)
def init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)