22__all__ = [
"init_fromDict",
"Functor",
"CompositeFunctor",
"mag_aware_eval",
23 "CustomFunctor",
"Column",
"Index",
"CoordColumn",
"RAColumn",
24 "DecColumn",
"HtmIndex20",
"fluxName",
"fluxErrName",
"Mag",
25 "MagErr",
"MagDiff",
"Color",
"DeconvolvedMoments",
"SdssTraceSize",
26 "PsfSdssTraceSizeDiff",
"HsmTraceSize",
"PsfHsmTraceSizeDiff",
27 "HsmFwhm",
"E1",
"E2",
"RadiusFromQuadrupole",
"LocalWcs",
28 "ComputePixelScale",
"ConvertPixelToArcseconds",
29 "ConvertPixelSqToArcsecondsSq",
"ReferenceBand",
"Photometry",
30 "NanoJansky",
"NanoJanskyErr",
"LocalPhotometry",
"LocalNanojansky",
31 "LocalNanojanskyErr",
"LocalDipoleMeanFlux",
32 "LocalDipoleMeanFluxErr",
"LocalDipoleDiffFlux",
33 "LocalDipoleDiffFluxErr",
"Ebv",
41from contextlib
import redirect_stdout
42from itertools
import product
44import astropy.units
as u
50from astropy.coordinates
import SkyCoord
51from lsst.daf.butler
import DeferredDatasetHandle
52from lsst.pipe.base
import InMemoryDatasetHandle
53from lsst.utils
import doImport
54from lsst.utils.introspection
import get_full_type_name
58 typeKey='functor', name=None):
59 """Initialize an object defined in a dictionary.
61 The object needs to be importable as f'{basePath}.{initDict[typeKey]}'.
62 The positional and keyword arguments (if any) are contained in "args" and
63 "kwargs" entries in the dictionary, respectively.
64 This is used in `~lsst.pipe.tasks.functors.CompositeFunctor.from_yaml` to
65 initialize a composite functor from a specification in a YAML file.
70 Dictionary describing object's initialization.
71 Must contain an entry keyed by ``typeKey`` that is the name of the
72 object, relative to ``basePath``.
74 Path relative to module in which ``initDict[typeKey]`` is defined.
76 Key of ``initDict`` that is the name of the object (relative to
79 initDict = initDict.copy()
81 pythonType = doImport(f
'{basePath}.{initDict.pop(typeKey)}')
83 if 'args' in initDict:
84 args = initDict.pop(
'args')
85 if isinstance(args, str):
88 element = pythonType(*args, **initDict)
89 except Exception
as e:
90 message = f
'Error in constructing functor "{name}" of type {pythonType.__name__} with args: {args}'
91 raise type(e)(message, e.args)
96 """Define and execute a calculation on a DataFrame or Handle holding a
99 The `__call__` method accepts either a `~pandas.DataFrame` object or a
100 `~lsst.daf.butler.DeferredDatasetHandle` or
101 `~lsst.pipe.base.InMemoryDatasetHandle`, and returns the
102 result of the calculation as a single column.
103 Each functor defines what columns are needed for the calculation, and only
104 these columns are read from the dataset handle.
106 The action of `__call__` consists of two steps: first, loading the
107 necessary columns from disk into memory as a `~pandas.DataFrame` object;
108 and second, performing the computation on this DataFrame and returning the
111 To define a new `Functor`, a subclass must define a `_func` method,
112 that takes a `~pandas.DataFrame` and returns result in a `~pandas.Series`.
113 In addition, it must define the following attributes:
115 * `_columns`: The columns necessary to perform the calculation
116 * `name`: A name appropriate for a figure axis label
117 * `shortname`: A name appropriate for use as a dictionary key
119 On initialization, a `Functor` should declare what band (``filt`` kwarg)
120 and dataset (e.g. ``'ref'``, ``'meas'``, ``'forced_src'``) it is intended
122 This enables the `_get_data` method to extract the proper columns from the
124 If not specified, the dataset will fall back on the `_defaultDataset`
126 If band is not specified and ``dataset`` is anything other than ``'ref'``,
127 then an error will be raised when trying to perform the calculation.
129 Originally, `Functor` was set up to expect datasets formatted like the
130 ``deepCoadd_obj`` dataset; that is, a DataFrame with a multi-level column
131 index, with the levels of the column index being ``band``, ``dataset``, and
133 It has since been generalized to apply to DataFrames without multi-level
134 indices and multi-level indices with just ``dataset`` and ``column``
136 In addition, the `_get_data` method that reads the columns from the
137 underlying data will return a DataFrame with column index levels defined by
138 the `_dfLevels` attribute; by default, this is ``column``.
140 The `_dfLevels` attributes should generally not need to be changed, unless
141 `_func` needs columns from multiple filters or datasets to do the
143 An example of this is the `~lsst.pipe.tasks.functors.Color` functor, for
144 which `_dfLevels = ('band', 'column')`, and `_func` expects the DataFrame
145 it gets to have those levels in the column index.
150 Band upon which to do the calculation.
153 Dataset upon which to do the calculation (e.g., 'ref', 'meas',
157 _defaultDataset =
'ref'
158 _dfLevels = (
'column',)
159 _defaultNoDup =
False
161 def __init__(self, filt=None, dataset=None, noDup=None):
165 self.
log = logging.getLogger(type(self).__name__)
169 """Do not explode by band if used on object table."""
170 if self.
_noDup is not None:
177 """Columns required to perform calculation."""
178 if not hasattr(self,
'_columns'):
179 raise NotImplementedError(
'Must define columns property or _columns attribute')
183 """Gets the names of the column index levels.
185 This should only be called in the context of a multilevel table.
190 The data to be read, can be a
191 `~lsst.daf.butler.DeferredDatasetHandle` or
192 `~lsst.pipe.base.InMemoryDatasetHandle`.
193 columnIndex (optional): pandas `~pandas.Index` object
194 If not passed, then it is read from the
195 `~lsst.daf.butler.DeferredDatasetHandle`
196 for `~lsst.pipe.base.InMemoryDatasetHandle`.
198 if columnIndex
is None:
199 columnIndex = data.get(component=
"columns")
200 return columnIndex.names
203 """Gets the content of each of the column levels for a multilevel
206 if columnIndex
is None:
207 columnIndex = data.get(component=
"columns")
209 columnLevels = columnIndex.names
211 level: list(np.unique(np.array([c
for c
in columnIndex])[:, i]))
212 for i, level
in enumerate(columnLevels)
214 return columnLevelNames
217 """Converts dictionary column specficiation to a list of columns."""
221 for i, lev
in enumerate(columnLevels):
223 if isinstance(colDict[lev], str):
224 new_colDict[lev] = [colDict[lev]]
226 new_colDict[lev] = colDict[lev]
228 new_colDict[lev] = columnIndex.levels[i]
230 levelCols = [new_colDict[lev]
for lev
in columnLevels]
231 cols = list(product(*levelCols))
232 colsAvailable = [col
for col
in cols
if col
in columnIndex]
236 """Returns columns needed by functor from multilevel dataset.
238 To access tables with multilevel column structure, the
239 `~lsst.daf.butler.DeferredDatasetHandle` or
240 `~lsst.pipe.base.InMemoryDatasetHandle` needs to be passed
241 either a list of tuples or a dictionary.
246 The data as either `~lsst.daf.butler.DeferredDatasetHandle`, or
247 `~lsst.pipe.base.InMemoryDatasetHandle`.
248 columnIndex (optional): pandas `~pandas.Index` object
249 Either passed or read in from
250 `~lsst.daf.butler.DeferredDatasetHandle`.
251 `returnTuple` : `bool`
252 If true, then return a list of tuples rather than the column
253 dictionary specification.
254 This is set to `True` by `CompositeFunctor` in order to be able to
255 combine columns from the various component functors.
258 if not isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
259 raise RuntimeError(f
"Unexpected data type. Got {get_full_type_name(data)}.")
261 if columnIndex
is None:
262 columnIndex = data.get(component=
"columns")
268 columnDict = {
'column': self.
columns,
270 if self.
filt is None:
272 if "band" in columnLevels:
274 columnDict[
"band"] = columnLevelNames[
"band"][0]
276 raise ValueError(f
"'filt' not set for functor {self.name}"
277 f
"(dataset {self.dataset}) "
279 "contains multiple filters in column index. "
280 "Set 'filt' or set 'dataset' to 'ref'.")
282 columnDict[
'band'] = self.
filt
285 return self.
_colsFromDict(columnDict, columnIndex=columnIndex)
290 raise NotImplementedError(
'Must define calculation on DataFrame')
293 """Return columnIndex."""
295 if isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
296 return data.get(component=
"columns")
301 """Retrieve DataFrame necessary for calculation.
303 The data argument can be a `~pandas.DataFrame`, a
304 `~lsst.daf.butler.DeferredDatasetHandle`, or
305 an `~lsst.pipe.base.InMemoryDatasetHandle`.
307 Returns a DataFrame upon which `self._func` can act.
311 if isinstance(data, pd.DataFrame):
312 _data = InMemoryDatasetHandle(data, storageClass=
"DataFrame")
313 elif isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
316 raise RuntimeError(f
"Unexpected type provided for data. Got {get_full_type_name(data)}.")
321 is_multiLevel = isinstance(columnIndex, pd.MultiIndex)
330 df = _data.get(parameters={
"columns": columns})
339 levelsToDrop = [n
for n
in df.columns.names
if n
not in self.
_dfLevels]
340 df.columns = df.columns.droplevel(levelsToDrop)
349 vals = self.
_func(df)
350 except Exception
as e:
351 self.
log.error(
"Exception in %s call: %s: %s", self.
namename, type(e).__name__, e)
359 """Computes difference between functor called on two different
360 DataFrame/Handle objects.
362 return self(data1, **kwargs) - self(data2, **kwargs)
365 return pd.Series(np.full(len(df), np.nan), index=df.index)
369 """Full name of functor (suitable for figure labels)."""
370 return NotImplementedError
374 """Short name of functor (suitable for column name/dict key)."""
379 """Perform multiple calculations at once on a catalog.
381 The role of a `CompositeFunctor` is to group together computations from
383 Instead of returning `~pandas.Series` a `CompositeFunctor` returns a
384 `~pandas.DataFrame`, with the column names being the keys of ``funcDict``.
386 The `columns` attribute of a `CompositeFunctor` is the union of all columns
387 in all the component functors.
389 A `CompositeFunctor` does not use a `_func` method itself; rather, when a
390 `CompositeFunctor` is called, all its columns are loaded at once, and the
391 resulting DataFrame is passed to the `_func` method of each component
393 This has the advantage of only doing I/O (reading from parquet file) once,
394 and works because each individual `_func` method of each component functor
395 does not care if there are *extra* columns in the DataFrame being passed;
396 only that it must contain *at least* the `columns` it expects.
398 An important and useful class method is `from_yaml`, which takes as an
399 argument the path to a YAML file specifying a collection of functors.
403 funcs : `dict` or `list`
404 Dictionary or list of functors.
405 If a list, then it will be converted into a dictonary according to the
406 `.shortname` attribute of each functor.
409 name =
"CompositeFunctor"
413 if type(funcs) == dict:
416 self.
funcDict = {f.shortname: f
for f
in funcs}
434 """Update the functor with new functors."""
435 if isinstance(new, dict):
437 elif isinstance(new, CompositeFunctor):
440 raise TypeError(
'Can only update with dictionary or CompositeFunctor.')
448 return list(set([x
for y
in [f.columns
for f
in self.
funcDict.values()]
for x
in y]))
458 f.multilevelColumns(data, returnTuple=
True, **kwargs)
for f
in self.
funcDict.values()
466 """Apply the functor to the data table.
471 The data represented as `~lsst.daf.butler.DeferredDatasetHandle`,
472 `~lsst.pipe.base.InMemoryDatasetHandle`, or `~pandas.DataFrame`.
473 The table or a pointer to a table on disk from which columns can
476 if isinstance(data, pd.DataFrame):
477 _data = InMemoryDatasetHandle(data, storageClass=
"DataFrame")
478 elif isinstance(data, (DeferredDatasetHandle, InMemoryDatasetHandle)):
481 raise RuntimeError(f
"Unexpected type provided for data. Got {get_full_type_name(data)}.")
485 if isinstance(columnIndex, pd.MultiIndex):
487 df = _data.get(parameters={
"columns": columns})
492 subdf = f._setLevels(
493 df[f.multilevelColumns(_data, returnTuple=
True, columnIndex=columnIndex)]
495 valDict[k] = f._func(subdf)
496 except Exception
as e:
498 "Exception in %s (funcs: %s) call: %s",
504 valDict[k] = f.fail(subdf)
511 valDict = {k: f._func(df)
for k, f
in self.
funcDict.items()}
514 for name, colVal
in valDict.items():
515 if len(colVal.shape) != 1:
516 raise RuntimeError(
"Transformed column '%s' is not the shape of a column. "
517 "It is shaped %s and type %s." % (name, colVal.shape, type(colVal)))
520 valDf = pd.concat(valDict, axis=1)
522 print([(k, type(v))
for k, v
in valDict.items()])
525 if kwargs.get(
'dropna',
False):
526 valDf = valDf.dropna(how=
'any')
532 if renameRules
is None:
534 for old, new
in renameRules:
535 if col.startswith(old):
536 col = col.replace(old, new)
542 filename = os.path.expandvars(filename)
543 with open(filename)
as f:
544 translationDefinition = yaml.safe_load(f)
546 return cls.
from_yaml(translationDefinition, **kwargs)
551 for func, val
in translationDefinition[
'funcs'].items():
554 if 'flag_rename_rules' in translationDefinition:
555 renameRules = translationDefinition[
'flag_rename_rules']
559 if 'calexpFlags' in translationDefinition:
560 for flag
in translationDefinition[
'calexpFlags']:
561 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'calexp')
563 if 'refFlags' in translationDefinition:
564 for flag
in translationDefinition[
'refFlags']:
567 if 'forcedFlags' in translationDefinition:
568 for flag
in translationDefinition[
'forcedFlags']:
569 funcs[cls.
renameCol(flag, renameRules)] =
Column(flag, dataset=
'forced_src')
571 if 'flags' in translationDefinition:
572 for flag
in translationDefinition[
'flags']:
575 return cls(funcs, **kwargs)
579 """Evaluate an expression on a DataFrame, knowing what the 'mag' function
582 Builds on `pandas.DataFrame.eval`, which parses and executes math on
587 df : ~pandas.DataFrame
588 DataFrame on which to evaluate expression.
594 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>)/log(10)', expr)
595 val = df.eval(expr_new)
596 except Exception
as e:
597 log.error(
"Exception in mag_aware_eval: %s: %s", type(e).__name__, e)
598 expr_new = re.sub(
r'mag\((\w+)\)',
r'-2.5*log(\g<1>_instFlux)/log(10)', expr)
599 val = df.eval(expr_new)
604 """Arbitrary computation on a catalog.
606 Column names (and thus the columns to be loaded from catalog) are found by
607 finding all words and trying to ignore all "math-y" words.
612 Expression to evaluate, to be parsed and executed by
613 `~lsst.pipe.tasks.functors.mag_aware_eval`.
615 _ignore_words = (
'mag',
'sin',
'cos',
'exp',
'log',
'sqrt')
627 flux_cols = re.findall(
r'mag\(\s*(\w+)\s*\)', self.
expr)
629 cols = [c
for c
in re.findall(
r'[a-zA-Z_]+', self.
expr)
if c
not in self.
_ignore_words]
632 if not re.search(
'_instFlux$', c):
633 cols.append(f
'{c}_instFlux')
638 return list(set([c
for c
in cols
if c
not in not_a_col]))
645 """Get column with a specified name."""
664 """Return the value of the index for each object."""
666 columns = [
'coord_ra']
667 _defaultDataset =
'ref'
671 return pd.Series(df.index, index=df.index)
675 """Base class for coordinate column, in degrees."""
684 output = df[self.
col] * 180 / np.pi
if self.
_radians else df[self.
col]
689 """Right Ascension, in degrees."""
694 super().
__init__(
'coord_ra', **kwargs)
697 return super().
__call__(catalog, **kwargs)
701 """Declination, in degrees."""
706 super().
__init__(
'coord_dec', **kwargs)
709 return super().
__call__(catalog, **kwargs)
713 """Uncertainty in Right Ascension, in degrees."""
718 super().
__init__(
'coord_raErr', **kwargs)
722 """Uncertainty in declination, in degrees."""
727 super().
__init__(
'coord_decErr', **kwargs)
731 """Coordinate covariance column, in degrees."""
737 super().
__init__(
'coord_ra_dec_Cov', **kwargs)
742 output = df[self.
col]*(180/np.pi)**2
if self.
_radians else df[self.
col]
747 """Compute the level 20 HtmIndex for the catalog.
751 This functor was implemented to satisfy requirements of old APDB interface
752 which required the ``pixelId`` column in DiaObject with HTM20 index.
753 The APDB interface had migrated to not need that information, but we keep
754 this class in case it may be useful for something else.
769 def computePixel(row):
778 return self.
pixelator.index(sphPoint.getVector())
780 return df.apply(computePixel, axis=1, result_type=
'reduce').astype(
'int64')
784 """Append _instFlux to the column name if it doesn't have it already."""
785 if not col.endswith(
'_instFlux'):
791 """Append _instFluxErr to the column name if it doesn't have it already."""
792 if not col.endswith(
'_instFluxErr'):
793 col +=
'_instFluxErr'
798 """Compute calibrated magnitude.
800 Returns the flux at mag=0.
801 The default ``fluxMag0`` is 63095734448.0194, which is default for HSC.
802 TO DO: This default should be made configurable in DM-21955.
804 This calculation hides warnings about invalid values and dividing by zero.
806 As with all functors, a ``dataset`` and ``filt`` kwarg should be provided
808 Unlike the default `Functor`, however, the default dataset for a `Mag` is
809 ``'meas'``, rather than ``'ref'``.
814 Name of flux column from which to compute magnitude.
815 Can be parseable by the `~lsst.pipe.tasks.functors.fluxName` function;
816 that is, you can pass ``'modelfit_CModel'`` instead of
817 ``'modelfit_CModel_instFlux'``, and it will understand.
819 _defaultDataset =
'meas'
833 with warnings.catch_warnings():
834 warnings.filterwarnings(
'ignore',
r'invalid value encountered')
835 warnings.filterwarnings(
'ignore',
r'divide by zero')
840 return f
'mag_{self.col}'
844 """Compute calibrated magnitude uncertainty.
849 Name of the flux column.
862 with warnings.catch_warnings():
863 warnings.filterwarnings(
'ignore',
r'invalid value encountered')
864 warnings.filterwarnings(
'ignore',
r'divide by zero')
866 x = df[fluxErrCol] / df[fluxCol]
868 magErr = (2.5 / np.log(10.)) * np.sqrt(x*x + y*y)
873 return super().name +
'_err'
877 """Functor to calculate magnitude difference."""
878 _defaultDataset =
'meas'
890 with warnings.catch_warnings():
891 warnings.filterwarnings(
'ignore',
r'invalid value encountered')
892 warnings.filterwarnings(
'ignore',
r'divide by zero')
893 return -2.5*np.log10(df[self.
col1]/df[self.
col2])
897 return f
'(mag_{self.col1} - mag_{self.col2})'
901 return f
'magDiff_{self.col1}_{self.col2}'
905 """Compute the color between two filters.
907 Computes color by initializing two different `Mag` functors based on the
908 ``col`` and filters provided, and then returning the difference.
910 This is enabled by the `_func` method expecting a DataFrame with a
911 multilevel column index, with both ``'band'`` and ``'column'``, instead of
912 just ``'column'``, which is the `Functor` default.
913 This is controlled by the `_dfLevels` attribute.
915 Also of note, the default dataset for `Color` is ``forced_src'``, whereas
916 for `Mag` it is ``'meas'``.
921 Name of the flux column from which to compute; same as would be passed
922 to `~lsst.pipe.tasks.functors.Mag`.
925 Filters from which to compute magnitude difference.
926 Color computed is ``Mag(filt2) - Mag(filt1)``.
928 _defaultDataset =
'forced_src'
929 _dfLevels = (
'band',
'column')
935 raise RuntimeError(
"Cannot compute Color for %s: %s - %s " % (col, filt2, filt1))
953 mag2 = self.mag2.
_func(df[self.filt2])
954 mag1 = self.mag1.
_func(df[self.filt1])
959 return [self.
mag1.col, self.
mag2.col]
966 return f
'{self.filt2} - {self.filt1} ({self.col})'
970 return f
"{self.col}_{self.filt2.replace('-', '')}m{self.filt1.replace('-', '')}"
974 """This functor subtracts the trace of the PSF second moments from the
975 trace of the second moments of the source.
977 If the HsmShapeAlgorithm measurement is valid, then these will be used for
979 Otherwise, the SdssShapeAlgorithm measurements will be used.
981 name =
'Deconvolved Moments'
982 shortname =
'deconvolvedMoments'
983 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
984 "ext_shapeHSM_HsmSourceMoments_yy",
985 "base_SdssShape_xx",
"base_SdssShape_yy",
986 "ext_shapeHSM_HsmPsfMoments_xx",
987 "ext_shapeHSM_HsmPsfMoments_yy")
990 """Calculate deconvolved moments."""
991 if "ext_shapeHSM_HsmSourceMoments_xx" in df.columns:
992 hsm = df[
"ext_shapeHSM_HsmSourceMoments_xx"] + df[
"ext_shapeHSM_HsmSourceMoments_yy"]
994 hsm = np.ones(len(df))*np.nan
995 sdss = df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]
996 if "ext_shapeHSM_HsmPsfMoments_xx" in df.columns:
997 psf = df[
"ext_shapeHSM_HsmPsfMoments_xx"] + df[
"ext_shapeHSM_HsmPsfMoments_yy"]
1002 raise RuntimeError(
'No psf shape parameter found in catalog')
1004 return hsm.where(np.isfinite(hsm), sdss) - psf
1008 """Functor to calculate the SDSS trace radius size for sources.
1010 The SDSS trace radius size is a measure of size equal to the square root of
1011 half of the trace of the second moments tensor measured with the
1012 SdssShapeAlgorithm plugin.
1013 This has units of pixels.
1015 name =
"SDSS Trace Size"
1016 shortname =
'sdssTrace'
1017 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy")
1020 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1025 """Functor to calculate the SDSS trace radius size difference (%) between
1026 the object and the PSF model.
1032 name =
"PSF - SDSS Trace Size"
1033 shortname =
'psf_sdssTrace'
1034 _columns = (
"base_SdssShape_xx",
"base_SdssShape_yy",
1035 "base_SdssShape_psf_xx",
"base_SdssShape_psf_yy")
1038 srcSize = np.sqrt(0.5*(df[
"base_SdssShape_xx"] + df[
"base_SdssShape_yy"]))
1039 psfSize = np.sqrt(0.5*(df[
"base_SdssShape_psf_xx"] + df[
"base_SdssShape_psf_yy"]))
1040 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1045 """Functor to calculate the HSM trace radius size for sources.
1047 The HSM trace radius size is a measure of size equal to the square root of
1048 half of the trace of the second moments tensor measured with the
1049 HsmShapeAlgorithm plugin.
1050 This has units of pixels.
1052 name =
'HSM Trace Size'
1053 shortname =
'hsmTrace'
1054 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1055 "ext_shapeHSM_HsmSourceMoments_yy")
1058 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1059 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1064 """Functor to calculate the HSM trace radius size difference (%) between
1065 the object and the PSF model.
1071 name =
'PSF - HSM Trace Size'
1072 shortname =
'psf_HsmTrace'
1073 _columns = (
"ext_shapeHSM_HsmSourceMoments_xx",
1074 "ext_shapeHSM_HsmSourceMoments_yy",
1075 "ext_shapeHSM_HsmPsfMoments_xx",
1076 "ext_shapeHSM_HsmPsfMoments_yy")
1079 srcSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmSourceMoments_xx"]
1080 + df[
"ext_shapeHSM_HsmSourceMoments_yy"]))
1081 psfSize = np.sqrt(0.5*(df[
"ext_shapeHSM_HsmPsfMoments_xx"]
1082 + df[
"ext_shapeHSM_HsmPsfMoments_yy"]))
1083 sizeDiff = 100*(srcSize - psfSize)/(0.5*(srcSize + psfSize))
1088 """Functor to calculate the PSF FWHM with second moments measured from the
1089 HsmShapeAlgorithm plugin.
1091 This is in units of arcseconds, and assumes the hsc_rings_v1 skymap pixel
1092 scale of 0.168 arcseconds/pixel.
1096 This conversion assumes the PSF is Gaussian, which is not always the case.
1098 name =
'HSM Psf FWHM'
1099 _columns = (
'ext_shapeHSM_HsmPsfMoments_xx',
'ext_shapeHSM_HsmPsfMoments_yy')
1102 SIGMA2FWHM = 2*np.sqrt(2*np.log(2))
1106 0.5*(df[
'ext_shapeHSM_HsmPsfMoments_xx'] + df[
'ext_shapeHSM_HsmPsfMoments_yy']))
1110 r"""Calculate :math:`e_1` ellipticity component for sources, defined as:
1113 e_1 &= (I_{xx}-I_{yy})/(I_{xx}+I_{yy})
1119 name =
"Distortion Ellipticity (e1)"
1120 shortname =
"Distortion"
1138 r"""Calculate :math:`e_2` ellipticity component for sources, defined as:
1141 e_2 &= 2I_{xy}/(I_{xx}+I_{yy})
1147 name =
"Ellipticity e2"
1164 """Calculate the radius from the quadrupole moments.
1166 This returns the fourth root of the determinant of the second moments
1167 tensor, which has units of pixels.
1190 """Computations using the stored localWcs."""
1191 name =
"LocalWcsOperations"
1206 """Compute the distance on the sphere from x2, y1 to x1, y1.
1210 x : `~pandas.Series`
1212 y : `~pandas.Series`
1214 cd11 : `~pandas.Series`
1215 [1, 1] element of the local Wcs affine transform.
1216 cd11 : `~pandas.Series`
1217 [1, 1] element of the local Wcs affine transform.
1218 cd12 : `~pandas.Series`
1219 [1, 2] element of the local Wcs affine transform.
1220 cd21 : `~pandas.Series`
1221 [2, 1] element of the local Wcs affine transform.
1222 cd22 : `~pandas.Series`
1223 [2, 2] element of the local Wcs affine transform.
1228 RA and dec conversion of x and y given the local Wcs.
1229 Returned units are in radians.
1232 return (x * cd11 + y * cd12, x * cd21 + y * cd22)
1235 """Compute the local pixel scale conversion.
1239 ra1 : `~pandas.Series`
1240 Ra of the first coordinate in radians.
1241 dec1 : `~pandas.Series`
1242 Dec of the first coordinate in radians.
1243 ra2 : `~pandas.Series`
1244 Ra of the second coordinate in radians.
1245 dec2 : `~pandas.Series`
1246 Dec of the second coordinate in radians.
1250 dist : `~pandas.Series`
1251 Distance on the sphere in radians.
1253 deltaDec = dec2 - dec1
1255 return 2 * np.arcsin(
1257 np.sin(deltaDec / 2) ** 2
1258 + np.cos(dec2) * np.cos(dec1) * np.sin(deltaRa / 2) ** 2))
1261 """Compute the distance on the sphere from x2, y1 to x1, y1.
1265 x1 : `~pandas.Series`
1267 y1 : `~pandas.Series`
1269 x2 : `~pandas.Series`
1271 y2 : `~pandas.Series`
1273 cd11 : `~pandas.Series`
1274 [1, 1] element of the local Wcs affine transform.
1275 cd11 : `~pandas.Series`
1276 [1, 1] element of the local Wcs affine transform.
1277 cd12 : `~pandas.Series`
1278 [1, 2] element of the local Wcs affine transform.
1279 cd21 : `~pandas.Series`
1280 [2, 1] element of the local Wcs affine transform.
1281 cd22 : `~pandas.Series`
1282 [2, 2] element of the local Wcs affine transform.
1286 Distance : `~pandas.Series`
1287 Arcseconds per pixel at the location of the local WC.
1296 """Compute the local pixel scale from the stored CDMatrix.
1308 """Compute the local pixel to scale conversion in arcseconds.
1312 cd11 : `~pandas.Series`
1313 [1, 1] element of the local Wcs affine transform in radians.
1314 cd11 : `~pandas.Series`
1315 [1, 1] element of the local Wcs affine transform in radians.
1316 cd12 : `~pandas.Series`
1317 [1, 2] element of the local Wcs affine transform in radians.
1318 cd21 : `~pandas.Series`
1319 [2, 1] element of the local Wcs affine transform in radians.
1320 cd22 : `~pandas.Series`
1321 [2, 2] element of the local Wcs affine transform in radians.
1325 pixScale : `~pandas.Series`
1326 Arcseconds per pixel at the location of the local WC.
1328 return 3600 * np.degrees(np.sqrt(np.fabs(cd11 * cd22 - cd12 * cd21)))
1338 """Convert a value in units of pixels to units of arcseconds."""
1356 return f
"{self.col}_asArcseconds"
1374 """Convert a value in units of pixels squared to units of arcseconds
1394 return f
"{self.col}_asArcsecondsSq"
1409 return df[self.
col] * pixScale * pixScale
1413 """Return the band used to seed multiband forced photometry.
1415 This functor is to be used on Object tables.
1416 It converts the boolean merge_measurements_{band} columns into a single
1417 string representing the first band for which merge_measurements_{band}
1420 Assumes the default priority order of i, r, z, y, g, u.
1422 name =
'Reference Band'
1423 shortname =
'refBand'
1427 return [
"merge_measurement_i",
1428 "merge_measurement_r",
1429 "merge_measurement_z",
1430 "merge_measurement_y",
1431 "merge_measurement_g",
1432 "merge_measurement_u"]
1434 def _func(self, df: pd.DataFrame) -> pd.Series:
1435 def getFilterAliasName(row):
1437 colName = row.idxmax()
1438 return colName.replace(
'merge_measurement_',
'')
1442 columns = [col
for col
in self.
columnscolumns if col
in df.columns]
1444 return df[columns].apply(getFilterAliasName, axis=1,
1445 result_type=
'reduce').astype(
'object')
1449 """Base class for Object table calibrated fluxes and magnitudes."""
1451 AB_FLUX_SCALE = (0 * u.ABmag).to_value(u.nJy)
1452 LOG_AB_FLUX_SCALE = 12.56
1453 FIVE_OVER_2LOG10 = 1.085736204758129569
1457 def __init__(self, colFlux, colFluxErr=None, **kwargs):
1473 return f
'mag_{self.col}'
1477 """Compute sqrt(a^2 + b^2) without under/overflow."""
1478 if np.abs(a) < np.abs(b):
1483 return np.abs(a) * np.sqrt(1. + q*q)
1486 """Convert instrumental flux to nanojanskys."""
1490 """Convert instrumental flux to AB magnitude."""
1491 with warnings.catch_warnings():
1492 warnings.filterwarnings(
'ignore',
r'invalid value encountered')
1493 warnings.filterwarnings(
'ignore',
r'divide by zero')
1494 return -2.5 * np.log10(dn/fluxMag0)
1497 """Convert instrumental flux error to nanojanskys."""
1498 retVal = self.
vhypot(dn * fluxMag0Err, dnErr * fluxMag0)
1503 """Convert instrumental flux error to AB magnitude error."""
1504 retVal = self.
dn2fluxErr(dn, dnErr, fluxMag0, fluxMag0Err) / self.
dn2flux(dn, fluxMag0)
1509 """Convert instrumental flux to nanojanskys."""
1515 """Convert instrumental flux error to nanojanskys."""
1522 return pd.Series(retArr, index=df.index)
1526 """Base class for calibrating the specified instrument flux column using
1527 the local photometric calibration.
1532 Name of the instrument flux column.
1533 instFluxErrCol : `str`
1534 Name of the assocated error columns for ``instFluxCol``.
1535 photoCalibCol : `str`
1536 Name of local calibration column.
1537 photoCalibErrCol : `str`
1538 Error associated with ``photoCalibCol``
1545 logNJanskyToAB = (1 * u.nJy).to_value(u.ABmag)
1560 """Convert instrument flux to nanojanskys.
1564 instFlux : `~numpy.ndarray` or `~pandas.Series`
1565 Array of instrument flux measurements.
1566 localCalib : `~numpy.ndarray` or `~pandas.Series`
1567 Array of local photometric calibration estimates.
1571 calibFlux : `~numpy.ndarray` or `~pandas.Series`
1572 Array of calibrated flux measurements.
1574 return instFlux * localCalib
1577 """Convert instrument flux to nanojanskys.
1581 instFlux : `~numpy.ndarray` or `~pandas.Series`
1582 Array of instrument flux measurements.
1583 instFluxErr : `~numpy.ndarray` or `~pandas.Series`
1584 Errors on associated ``instFlux`` values.
1585 localCalib : `~numpy.ndarray` or `~pandas.Series`
1586 Array of local photometric calibration estimates.
1587 localCalibErr : `~numpy.ndarray` or `~pandas.Series`
1588 Errors on associated ``localCalib`` values.
1592 calibFluxErr : `~numpy.ndarray` or `~pandas.Series`
1593 Errors on calibrated flux measurements.
1595 return np.hypot(instFluxErr * localCalib, instFlux * localCalibErr)
1598 """Convert instrument flux to nanojanskys.
1602 instFlux : `~numpy.ndarray` or `~pandas.Series`
1603 Array of instrument flux measurements.
1604 localCalib : `~numpy.ndarray` or `~pandas.Series`
1605 Array of local photometric calibration estimates.
1609 calibMag : `~numpy.ndarray` or `~pandas.Series`
1610 Array of calibrated AB magnitudes.
1615 """Convert instrument flux err to nanojanskys.
1619 instFlux : `~numpy.ndarray` or `~pandas.Series`
1620 Array of instrument flux measurements.
1621 instFluxErr : `~numpy.ndarray` or `~pandas.Series`
1622 Errors on associated ``instFlux`` values.
1623 localCalib : `~numpy.ndarray` or `~pandas.Series`
1624 Array of local photometric calibration estimates.
1625 localCalibErr : `~numpy.ndarray` or `~pandas.Series`
1626 Errors on associated ``localCalib`` values.
1630 calibMagErr: `~numpy.ndarray` or `~pandas.Series`
1631 Error on calibrated AB magnitudes.
1638 """Compute calibrated fluxes using the local calibration value.
1640 This returns units of nanojanskys.
1649 return f
'flux_{self.instFluxCol}'
1656 """Compute calibrated flux errors using the local calibration value.
1658 This returns units of nanojanskys.
1668 return f
'fluxErr_{self.instFluxCol}'
1676 """Compute absolute mean of dipole fluxes.
1682 LocalDipoleMeanFluxErr
1684 LocalDipoleDiffFluxErr
1714 return f
'dipMeanFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1722 """Compute the error on the absolute mean of dipole fluxes.
1730 LocalDipoleDiffFluxErr
1744 return f
'dipMeanFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1755 """Compute the absolute difference of dipole fluxes.
1757 Calculated value is (abs(pos) - abs(neg)).
1764 LocalDipoleMeanFluxErr
1765 LocalDipoleDiffFluxErr
1776 return f
'dipDiffFlux_{self.instFluxPosCol}_{self.instFluxNegCol}'
1784 """Compute the error on the absolute difference of dipole fluxes.
1791 LocalDipoleMeanFluxErr
1806 return f
'dipDiffFluxErr_{self.instFluxPosCol}_{self.instFluxNegCol}'
1817 """Compute E(B-V) from dustmaps.sfd."""
1818 _defaultDataset =
'ref'
1825 with open(os.devnull,
"w")
as devnull:
1826 with redirect_stdout(devnull):
1827 from dustmaps.sfd
import SFDQuery
1833 coords = SkyCoord(df[
'coord_ra'].values * u.rad, df[
'coord_dec'].values * u.rad)
1834 ebv = self.
sfd(coords)
1837 return pd.Series(ebv, index=df.index).astype(
'float64')
__init__(self, col, filt2, filt1, **kwargs)
multilevelColumns(self, parq, **kwargs)
__init__(self, col, **kwargs)
multilevelColumns(self, data, **kwargs)
__call__(self, data, **kwargs)
from_file(cls, filename, **kwargs)
renameCol(cls, col, renameRules)
from_yaml(cls, translationDefinition, **kwargs)
__init__(self, funcs, **kwargs)
pixelScaleArcseconds(self, cd11, cd12, cd21, cd22)
__init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
__init__(self, col, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
__init__(self, col, **kwargs)
__init__(self, expr, **kwargs)
__call__(self, catalog, **kwargs)
__init__(self, colXX, colXY, colYY, **kwargs)
__init__(self, colXX, colXY, colYY, **kwargs)
_func(self, df, dropna=True)
_get_columnIndex(self, data)
multilevelColumns(self, data, columnIndex=None, returnTuple=False)
__call__(self, data, dropna=False)
_get_data_columnLevels(self, data, columnIndex=None)
_colsFromDict(self, colDict, columnIndex=None)
difference(self, data1, data2, **kwargs)
_get_data_columnLevelNames(self, data, columnIndex=None)
__init__(self, filt=None, dataset=None, noDup=None)
__init__(self, ra, dec, **kwargs)
__init__(self, instFluxPosCol, instFluxNegCol, instFluxPosErrCol, instFluxNegErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
instFluxToNanojansky(self, instFlux, localCalib)
instFluxErrToNanojanskyErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
instFluxToMagnitude(self, instFlux, localCalib)
__init__(self, instFluxCol, instFluxErrCol, photoCalibCol, photoCalibErrCol, **kwargs)
instFluxErrToMagnitudeErr(self, instFlux, instFluxErr, localCalib, localCalibErr)
computeSkySeparation(self, ra1, dec1, ra2, dec2)
__init__(self, colCD_1_1, colCD_1_2, colCD_2_1, colCD_2_2, **kwargs)
computeDeltaRaDec(self, x, y, cd11, cd12, cd21, cd22)
getSkySeparationFromPixel(self, x1, y1, x2, y2, cd11, cd12, cd21, cd22)
__init__(self, col1, col2, **kwargs)
__init__(self, *args, **kwargs)
__init__(self, col, **kwargs)
dn2flux(self, dn, fluxMag0)
__init__(self, colFlux, colFluxErr=None, **kwargs)
dn2mag(self, dn, fluxMag0)
dn2MagErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
dn2fluxErr(self, dn, dnErr, fluxMag0, fluxMag0Err)
__call__(self, catalog, **kwargs)
__init__(self, colXX, colXY, colYY, **kwargs)
pd.Series _func(self, pd.DataFrame df)
init_fromDict(initDict, basePath='lsst.pipe.tasks.functors', typeKey='functor', name=None)
mag_aware_eval(df, expr, log)