Coverage for python/lsst/analysis/tools/atools/diffMatched.py: 20%
283 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-16 04:38 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-16 04:38 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "ReferenceGalaxySelector",
25 "ReferenceObjectSelector",
26 "ReferenceStarSelector",
27 "MatchedRefCoaddToolBase",
28 "MatchedRefCoaddDiffTool",
29 "MatchedRefCoaddDiffColorTool",
30 "MatchedRefCoaddDiffMagTool",
31 "MatchedRefCoaddDiffPositionTool",
32)
34import copy
35from abc import abstractmethod
37import astropy.units as u
38from lsst.pex.config import DictField, Field
40from ..actions.vector import (
41 CalcBinnedStatsAction,
42 ColorDiff,
43 ColorError,
44 ConstantValue,
45 CosVector,
46 DivideVector,
47 DownselectVector,
48 LoadVector,
49 MultiplyVector,
50 SubtractVector,
51)
52from ..actions.vector.selectors import RangeSelector, ThresholdSelector, VectorSelector
53from ..interfaces import KeyedData, Vector
54from .genericBuild import MagnitudeXTool
55from .genericMetricAction import StructMetricAction
56from .genericPlotAction import StructPlotAction
57from .genericProduce import MagnitudeScatterPlot
60class ReferenceGalaxySelector(ThresholdSelector):
61 """A selector that selects galaxies from a catalog with a
62 boolean column identifying unresolved sources.
63 """
65 def __call__(self, data: KeyedData, **kwargs) -> Vector:
66 result = super().__call__(data=data, **kwargs)
67 if self.plotLabelKey:
68 self._addValueToPlotInfo("true galaxies", **kwargs)
69 return result
71 def setDefaults(self):
72 super().setDefaults()
73 self.op = "eq"
74 self.threshold = 0
75 self.plotLabelKey = "Selection: Galaxies"
76 self.vectorKey = "refcat_is_pointsource"
79class ReferenceObjectSelector(RangeSelector):
80 """A selector that selects all objects from a catalog with a
81 boolean column identifying unresolved sources.
82 """
84 def setDefaults(self):
85 super().setDefaults()
86 self.minimum = 0
87 self.vectorKey = "refcat_is_pointsource"
90class ReferenceStarSelector(ThresholdSelector):
91 """A selector that selects stars from a catalog with a
92 boolean column identifying unresolved sources.
93 """
95 def __call__(self, data: KeyedData, **kwargs) -> Vector:
96 result = super().__call__(data=data, **kwargs)
97 if self.plotLabelKey:
98 self._addValueToPlotInfo("true stars", **kwargs)
99 return result
101 def setDefaults(self):
102 super().setDefaults()
103 self.op = "eq"
104 self.plotLabelKey = "Selection: Stars"
105 self.threshold = 1
106 self.vectorKey = "refcat_is_pointsource"
109class MatchedRefCoaddToolBase(MagnitudeXTool):
110 """Base tool for matched-to-reference metrics/plots on coadds.
112 Metrics/plots are expected to use the reference magnitude and
113 require separate star/galaxy/all source selections.
115 Notes
116 -----
117 The tool does not use a standard coadd flag selector, because
118 it is expected that the matcher has been configured to select
119 appropriate candidates (and stores a match_candidate column).
120 """
122 def setDefaults(self):
123 super().setDefaults()
124 self.mag_x = "ref_matched"
125 self.process.buildActions.allSelector = ReferenceObjectSelector()
126 self.process.buildActions.galaxySelector = ReferenceGalaxySelector()
127 self.process.buildActions.starSelector = ReferenceStarSelector()
129 def finalize(self):
130 super().finalize()
131 self._set_flux_default("mag_x")
134class MatchedRefCoaddDiffTool(MatchedRefCoaddToolBase):
135 """Base tool for generic diffs between reference and measured values."""
137 compute_chi = Field[bool](
138 default=False,
139 doc="Whether to compute scaled flux residuals (chi) instead of magnitude differences",
140 )
141 # These are optional because validate can be called before finalize
142 # Validate should not fail in that case if it would otherwise succeed
143 name_prefix = Field[str](doc="Prefix for metric key", default=None, optional=True)
144 unit = Field[str](doc="Astropy unit of y-axis values", default=None, optional=True)
146 _mag_low_min: int = 15
147 _mag_low_max: int = 27
148 _mag_interval: int = 1
150 _names = {"stars": "star", "galaxies": "galaxy", "all": "all"}
151 _types = ("unresolved", "resolved", "all")
153 def _set_actions(self, suffix=None):
154 if suffix is None:
155 suffix = ""
157 for name_lower, name_singular in self._names.items():
158 name = name_lower.capitalize()
159 key = f"y{name}{suffix}"
160 setattr(
161 self.process.filterActions,
162 key,
163 DownselectVector(
164 vectorKey=f"diff{suffix}", selector=VectorSelector(vectorKey=f"{name_singular}Selector")
165 ),
166 )
168 for minimum in range(self._mag_low_min, self._mag_low_max + 1):
169 setattr(
170 self.process.calculateActions,
171 f"{name_lower}{minimum}{suffix}",
172 CalcBinnedStatsAction(
173 key_vector=key,
174 selector_range=RangeSelector(
175 vectorKey=key,
176 minimum=minimum,
177 maximum=minimum + self._mag_interval,
178 ),
179 return_minmax=False,
180 ),
181 )
183 def configureMetrics(
184 self,
185 unit: str | None = None,
186 name_prefix: str | None = None,
187 name_suffix: str = "_ref_mag{minimum}",
188 attr_suffix: str | None = None,
189 unit_select: str = "mag",
190 ):
191 """Configure metric actions and return units.
193 Parameters
194 ----------
195 unit : `str`
196 The (astropy) unit of the summary statistic metrics.
197 name_prefix : `str`
198 The prefix for the action (column) name.
199 name_suffix : `str`
200 The suffix for the action (column) name.
201 attr_suffix : `str`
202 The suffix for the attribute to assign the action to.
203 unit_select : `str`
204 The (astropy) unit of the selection (x-axis) column. Default "mag".
206 Returns
207 -------
208 units : `dict` [`str`, `str`]
209 A dict of the unit (value) for each metric name (key)
210 """
211 if unit is None:
212 unit = self.unit if self.unit is not None else ""
213 if name_prefix is None:
214 name_prefix = self.name_prefix if self.name_prefix is not None else ""
215 if attr_suffix is None:
216 attr_suffix = ""
218 if unit_select is None:
219 unit_select = "mag"
221 key_flux = self.config_mag_x.key_flux
223 units = {}
224 for name, name_class in zip(self._names, self._types):
225 name_capital = name.capitalize()
226 x_key = f"x{name_capital}"
228 for minimum in range(self._mag_low_min, self._mag_low_max + 1):
229 action = getattr(self.process.calculateActions, f"{name}{minimum}{attr_suffix}")
230 action.selector_range = RangeSelector(
231 vectorKey=x_key,
232 minimum=minimum,
233 maximum=minimum + self._mag_interval,
234 )
236 action.name_prefix = name_prefix.format(
237 key_flux=key_flux,
238 name_class=name_class,
239 )
240 if self.parameterizedBand:
241 action.name_prefix = f"{{band}}_{action.name_prefix}"
242 action.name_suffix = name_suffix.format(minimum=minimum)
244 units.update(
245 {
246 action.name_median: unit,
247 action.name_sigmaMad: unit,
248 action.name_count: "count",
249 action.name_select_median: unit_select,
250 }
251 )
252 return units
254 @property
255 def config_mag_y(self):
256 """Return the y-axis magnitude config.
258 Although tools may not end up using any flux measures in metrics or
259 plots, this should still be set to the flux measure that was matched
260 or selected against in the catalog not used for the x-axis."""
261 mag_y = self.get_key_flux_y()
262 if mag_y not in self.fluxes:
263 raise KeyError(f"{mag_y=} not in {self.fluxes}; was finalize called?")
264 # This is a logic error: it shouldn't be called before finalize
265 assert mag_y in self.fluxes
266 return self.fluxes[mag_y]
268 @abstractmethod
269 def get_key_flux_y(self) -> str:
270 """Return the key for the y-axis flux measure."""
271 raise NotImplementedError("subclasses must implement get_key_flux_y")
273 def setDefaults(self):
274 super().setDefaults()
275 self._set_actions()
278class MatchedRefCoaddDiffColorTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot):
279 """Tool for diffs between reference and measured coadd mags.
281 Notes
282 -----
283 Since this tool requires at least two bands, it is essentially impossible
284 to call on its own.
285 """
287 mag_y1 = Field[str](default="cmodel_err", doc="Flux field for first magnitude")
288 mag_y2 = Field[str](
289 doc="Flux field for second magnitude (to subtract from first); default same as first",
290 default=None,
291 optional=True,
292 )
293 bands = DictField[str, str](
294 doc="Bands for colors. ",
295 default={"u": "g", "g": "r,i", "r": "i", "i": "z", "z": "y"},
296 )
297 band_separator = Field[str](default=",", doc="Separator for multiple bands")
299 def _split_bands(self, band_list: str):
300 return band_list.split(self.band_separator)
302 def finalize(self):
303 # Check if it has already been finalized
304 if not hasattr(self.process.buildActions, "diff_0"):
305 if self.mag_y2 is None:
306 self.mag_y2 = self.mag_y1
307 # Ensure mag_y1/2 are set before any plot finalizes
308 # This may result in duplicate actions but these are just plain
309 # column selectors so that's not a serious problem
310 bands = {band1: self._split_bands(band2_list) for band1, band2_list in self.bands.items()}
311 n_bands = 0
313 for band1, band2_list in bands.items():
314 for band2 in band2_list:
315 mag_y1 = f"mag_y_{band1}"
316 mag_y2 = f"mag_y_{band2}"
317 mag_x1 = f"mag_x_{band1}"
318 mag_x2 = f"mag_x_{band2}"
319 self._set_flux_default(mag_y1, band=band1, name_mag=self.mag_y1)
320 self._set_flux_default(mag_y2, band=band2, name_mag=self.mag_y2)
321 self._set_flux_default(mag_x1, band=band1, name_mag=self.mag_x)
322 self._set_flux_default(mag_x2, band=band2, name_mag=self.mag_x)
323 n_bands += 1
325 self.suffixes_y_finalize = [f"_{idx}" for idx in range(n_bands)]
326 super().finalize()
328 self.unit = "" if self.compute_chi else "mmag"
329 subtype = "chi" if self.compute_chi else "diff"
331 metric_base = self.produce.metric
332 plot_base = self.produce.plot
334 actions_metric = {}
335 actions_plot = {}
337 config_mag_x = self.config_mag_x
338 config_mag_y = self.config_mag_y
339 name_short_x = config_mag_x.name_flux_short
340 name_short_y = config_mag_y.name_flux_short
342 idx = 0
343 for band1, band2_list in bands.items():
344 for band2 in band2_list:
345 name_color = f"{band1}_minus_{band2}"
346 # Keep this index-based to simplify finalize
347 suffix_y = f"_{idx}"
348 self._set_actions(suffix=suffix_y)
349 metric = copy.copy(metric_base)
350 self.name_prefix = (
351 f"photom_{name_short_y}_vs_{name_short_x}_color_{name_color}"
352 f"_{subtype}_{{name_class}}_"
353 )
354 metric.units = self.configureMetrics(attr_suffix=suffix_y)
355 plot = copy.copy(plot_base)
357 plot.suffix_y = suffix_y
358 plot.suffix_stat = suffix_y
360 mag_y1 = f"{self.mag_y1}_{band1}"
361 mag_y2 = f"{self.mag_y2}_{band2}"
362 mag_x1 = f"{self.mag_x}_{band1}"
363 mag_x2 = f"{self.mag_x}_{band2}"
365 diff = ColorDiff(
366 color1_flux1=getattr(self.process.buildActions, f"flux_{mag_y1}"),
367 color1_flux2=getattr(self.process.buildActions, f"flux_{mag_y2}"),
368 color2_flux1=getattr(self.process.buildActions, f"flux_{mag_x1}"),
369 color2_flux2=getattr(self.process.buildActions, f"flux_{mag_x2}"),
370 )
372 if self.compute_chi:
373 diff = DivideVector(
374 actionA=diff,
375 actionB=ColorError(
376 flux_err1=DivideVector(
377 actionA=getattr(self.process.buildActions, f"flux_err_{mag_y1}"),
378 actionB=getattr(self.process.buildActions, f"flux_{mag_y1}"),
379 ),
380 flux_err2=DivideVector(
381 actionA=getattr(self.process.buildActions, f"flux_err_{mag_y2}"),
382 actionB=getattr(self.process.buildActions, f"flux_{mag_y2}"),
383 ),
384 ),
385 )
386 setattr(self.process.buildActions, f"diff{plot.suffix_y}", diff)
388 label = f"({band1} - {band2}) ({config_mag_y.name_flux} - {config_mag_x.name_flux})"
389 label = f"chi = ({label})/error" if self.compute_chi else f"{label} (mmag)"
390 plot.yAxisLabel = label
391 actions_metric[name_color] = metric
392 actions_plot[name_color] = plot
393 idx += 1
394 action_metric = StructMetricAction()
395 for name_action, action in actions_metric.items():
396 setattr(action_metric.actions, name_action, action)
397 self.produce.metric = action_metric
398 action_plot = StructPlotAction()
399 for name_action, action in actions_plot.items():
400 setattr(action_plot.actions, name_action, action)
401 self.produce.plot = action_plot
403 def get_key_flux_y(self) -> str:
404 return self.mag_y1
406 def setDefaults(self):
407 # skip MatchedRefCoaddDiffTool.setDefaults's _setActions call
408 MatchedRefCoaddToolBase.setDefaults(self)
409 MagnitudeScatterPlot.setDefaults(self)
411 def validate(self):
412 super().validate()
413 errors = []
414 for band1, band2_list in self.bands.items():
415 bands = self._split_bands(band2_list)
416 if len(set(bands)) != len(bands):
417 errors.append(f"value={band2_list} is not a set for key={band1}")
418 if errors:
419 raise ValueError("\n".join(errors))
422class MatchedRefCoaddDiffMagTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot):
423 """Tool for diffs between reference and measured coadd mags."""
425 mag_y = Field[str](default="cmodel_err", doc="Flux (magnitude) field to difference against ref")
427 def finalize(self):
428 # Check if it has already been finalized
429 if not hasattr(self.process.buildActions, "diff"):
430 # Ensure mag_y is set before any plot finalizes
431 self._set_flux_default("mag_y")
432 super().finalize()
433 name_short_x = self.config_mag_x.name_flux_short
434 name_short_y = self.config_mag_y.name_flux_short
436 prefix_action = "flux" if self.compute_chi else "mag"
437 action_diff = SubtractVector(
438 actionA=getattr(self.process.buildActions, f"{prefix_action}_{self.mag_x}"),
439 actionB=getattr(self.process.buildActions, f"{prefix_action}_{self.mag_y}"),
440 )
442 if self.compute_chi:
443 key_err = f"flux_err_{self.mag_y}"
444 action_err = (
445 getattr(self.process.buildActions, key_err)
446 if hasattr(self.process.buildActions, key_err)
447 else getattr(self.process.buildActions, f"flux_err_{self.mag_x}")
448 )
449 self.process.buildActions.diff = DivideVector(actionA=action_diff, actionB=action_err)
450 else:
451 # set to mmag
452 self.process.buildActions.diff = MultiplyVector(
453 actionA=action_diff,
454 actionB=ConstantValue(value=1000.0),
455 )
456 if not self.produce.plot.yAxisLabel:
457 label = f"{self.config_mag_y.name_flux} - {self.config_mag_x.name_flux}"
458 self.produce.plot.yAxisLabel = (
459 f"chi = ({label})/error" if self.compute_chi else f"{label} (mmag)"
460 )
461 if self.unit is None:
462 self.unit = "" if self.compute_chi else "mmag"
463 if self.name_prefix is None:
464 subtype = "chi" if self.compute_chi else "diff"
465 self.name_prefix = f"photom_{name_short_y}_vs_{name_short_x}_mag_{subtype}_{{name_class}}_"
466 if not self.produce.metric.units:
467 self.produce.metric.units = self.configureMetrics()
469 def get_key_flux_y(self) -> str:
470 return self.mag_y
473class MatchedRefCoaddDiffPositionTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot):
474 """Tool for diffs between reference and measured coadd positions."""
476 coord_label = Field[str](
477 doc="The plot label for the astrometric variable (default coord_meas)",
478 optional=True,
479 default=None,
480 )
481 coord_meas = Field[str](
482 doc="The key for measured values of the astrometric variable",
483 optional=False,
484 )
485 coord_ref = Field[str](
486 doc="The key for reference values of the astrometric variable",
487 optional=False,
488 )
489 coord_ref_cos = Field[str](
490 doc="The key for reference values of the cosine correction astrometric variable"
491 " (i.e. dec if coord_meas is RA)",
492 default=None,
493 optional=True,
494 )
495 coord_ref_cos_unit = Field[str](
496 doc="astropy unit of coord_ref_cos",
497 default="deg",
498 optional=True,
499 )
500 mag_sn = Field[str](default="cmodel_err", doc="Flux (magnitude) field to use for S/N binning on plot")
501 # Default coords are in degrees and we want mas
502 scale_factor = Field[float](
503 doc="The factor to multiply distances by (e.g. the pixel scale if coordinates have pixel units or "
504 "the desired spherical coordinate unit conversion otherwise)",
505 default=3600000,
506 )
508 def finalize(self):
509 # Check if it has already been finalized
510 if not hasattr(self.process.buildActions, "diff"):
511 # Set before MagnitudeScatterPlot.finalize to undo PSF default.
512 # Matched ref tables may not have PSF fluxes, or prefer CModel.
513 self._set_flux_default("mag_sn")
514 super().finalize()
515 name = self.coord_label if self.coord_label else self.coord_meas
516 self.process.buildActions.pos_meas = LoadVector(vectorKey=self.coord_meas)
517 self.process.buildActions.pos_ref = LoadVector(vectorKey=self.coord_ref)
518 name_short_x = self.config_mag_x.name_flux_short
519 name_short_y = self.config_mag_y.name_flux_short
521 if self.compute_chi:
522 self.process.buildActions.diff = DivideVector(
523 actionA=SubtractVector(
524 actionA=self.process.buildActions.pos_meas,
525 actionB=self.process.buildActions.pos_ref,
526 ),
527 actionB=LoadVector(vectorKey=f"{self.process.buildActions.pos_meas.vectorKey}Err"),
528 )
529 else:
530 factor = ConstantValue(value=self.scale_factor)
531 if self.coord_ref_cos:
532 factor_cos = u.Unit(self.coord_ref_cos_unit).to(u.rad)
533 factor = MultiplyVector(
534 actionA=factor,
535 actionB=CosVector(
536 actionA=MultiplyVector(
537 actionA=ConstantValue(value=factor_cos),
538 actionB=LoadVector(vectorKey=self.coord_meas),
539 )
540 ),
541 )
542 self.process.buildActions.diff = MultiplyVector(
543 actionA=factor,
544 actionB=SubtractVector(
545 actionA=self.process.buildActions.pos_meas,
546 actionB=self.process.buildActions.pos_ref,
547 ),
548 )
549 if self.unit is None:
550 self.unit = "" if self.compute_chi else "mas"
551 if self.name_prefix is None:
552 subtype = "chi" if self.compute_chi else "diff"
553 coord_prefix = "" if "coord" in self.coord_meas else "coord_"
554 self.name_prefix = (
555 f"astrom_{name_short_y}_vs_{name_short_x}_{coord_prefix}{self.coord_meas}_{subtype}"
556 f"_{{name_class}}_"
557 )
558 if not self.produce.metric.units:
559 self.produce.metric.units = self.configureMetrics()
560 if not self.produce.plot.yAxisLabel:
561 label = f"({name_short_y} - {name_short_x})"
562 coord_suffix = "" if "coord" in name else " coord"
563 self.produce.plot.yAxisLabel = (
564 f"chi = ({label} {name}{coord_suffix})/error"
565 if self.compute_chi
566 else f"{label} {name}{coord_suffix} ({self.unit})"
567 )
569 def get_key_flux_y(self) -> str:
570 return self.mag_sn
573class MatchedRefCoaddDiffDistanceTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot):
574 """Tool for distances between matched reference and measured coadd
575 objects."""
577 key_dist = Field[str](default="match_distance", doc="Distance field key")
578 key_dist_err = Field[str](default="match_distanceErr", doc="Distance error field key")
579 mag_sn = Field[str](default="cmodel_err", doc="Flux (magnitude) field to use for S/N binning on plot")
580 # Default coords are in degrees and we want mas
581 scale_factor = Field[float](
582 doc="The factor to multiply distances by (e.g. the pixel scale if coordinates have pixel units or "
583 "the desired spherical coordinate unit conversion otherwise)",
584 default=3600000,
585 )
587 def finalize(self):
588 # Check if it has already been finalized
589 if not hasattr(self.process.buildActions, "diff"):
590 # Set before MagnitudeScatterPlot.finalize to undo PSF default.
591 # Matched ref tables may not have PSF fluxes, or prefer CModel.
592 self._set_flux_default("mag_sn")
593 super().finalize()
595 name_short_x = self.config_mag_x.name_flux_short
596 name_short_y = self.config_mag_y.name_flux_short
598 self.process.buildActions.dist = LoadVector(vectorKey=self.key_dist)
599 if self.compute_chi:
600 self.process.buildActions.diff = DivideVector(
601 actionA=self.process.buildActions.dist,
602 actionB=LoadVector(vectorKey=self.key_dist_err),
603 )
604 else:
605 self.process.buildActions.diff = MultiplyVector(
606 actionA=ConstantValue(value=self.scale_factor),
607 actionB=self.process.buildActions.dist,
608 )
609 if self.unit is None:
610 self.unit = "" if self.compute_chi else "mas"
611 if self.name_prefix is None:
612 subtype = "chi" if self.compute_chi else "diff"
613 self.name_prefix = f"astrom_dist_{{name_class}}_{subtype}_"
614 self.name_prefix = f"astrom_{name_short_y}_vs_{name_short_x}_dist_{subtype}_{{name_class}}_"
615 if not self.produce.metric.units:
616 self.produce.metric.units = self.configureMetrics()
617 if not self.produce.plot.yAxisLabel:
618 label = f"({name_short_y} - {name_short_x}) distance"
619 self.produce.plot.yAxisLabel = (
620 f"chi = {label}/error" if self.compute_chi else f"{label} ({self.unit})"
621 )
623 def get_key_flux_y(self) -> str:
624 return self.mag_sn