22__all__ = [
'ConvertCatalogCoordinatesConfig',
'MatchProbabilisticConfig',
'MatcherProbabilistic']
26from dataclasses
import dataclass
30from scipy.spatial
import cKDTree
32from typing
import Callable, Set
34logger_default = logging.getLogger(__name__)
37def _mul_column(column: np.array, value: float):
38 if value
is not None and value != 1:
43def _radec_to_xyz(ra, dec):
44 """Convert input ra/dec coordinates to spherical unit vectors.
48 ra, dec: `numpy.ndarray`
49 Arrays of right ascension/declination in degrees.
53 vectors : `numpy.ndarray`, (N, 3)
56 if ra.size != dec.size:
57 raise ValueError(
'ra and dec must be same size')
59 decs = np.radians(dec)
60 vectors = np.empty((ras.size, 3))
62 sin_dec = np.sin(np.pi / 2 - decs)
63 vectors[:, 0] = sin_dec * np.cos(ras)
64 vectors[:, 1] = sin_dec * np.sin(ras)
65 vectors[:, 2] = np.cos(np.pi / 2 - decs)
72 """Store frequently-reference (meta)data relevant for matching a catalog.
76 catalog : `pandas.DataFrame`
77 A pandas catalog to store extra information for.
78 select : `numpy.array`
79 A numpy boolean array of the same length
as catalog to be used
for
86 coordinate_factor: float = None
88 def __init__(self, catalog: pd.DataFrame, select: np.array =
None, coordinate_factor: float =
None):
90 self.
select = np.ones(self.
n, dtype=bool)
if select
is None else select
91 self.
indices = np.flatnonzero(select)
if select
is not None else np.arange(self.
n)
95@dataclass(frozen=True)
97 """A catalog with sources with coordinate columns in some standard format/units.
99 catalog : `pandas.DataFrame`
100 A catalog with comparable coordinate columns.
101 column_coord1 : `str`
102 The first spatial coordinate column name.
103 column_coord2 : `str`
104 The second spatial coordinate column name.
105 coord1 : `numpy.array`
106 The first spatial coordinate values.
107 coord2 : `numpy.array`
108 The second spatial coordinate values.
109 extras : `CatalogExtras`
110 Extra cached (meta)data
for the `catalog`.
112 catalog: pd.DataFrame
117 extras: CatalogExtras
121 """Configuration for the MatchProbabilistic matcher.
123 column_ref_coord1 = pexConfig.Field(
126 doc=
'The reference table column for the first spatial coordinate (usually x or ra).',
128 column_ref_coord2 = pexConfig.Field(
131 doc=
'The reference table column for the second spatial coordinate (usually y or dec).'
132 'Units must match column_ref_coord1.',
134 column_target_coord1 = pexConfig.Field(
137 doc=
'The target table column for the first spatial coordinate (usually x or ra).'
138 'Units must match column_ref_coord1.',
140 column_target_coord2 = pexConfig.Field(
143 doc=
'The target table column for the second spatial coordinate (usually y or dec).'
144 'Units must match column_ref_coord2.',
146 coords_spherical = pexConfig.Field(
149 doc=
'Whether column_*_coord[12] are spherical coordinates (ra/dec) or not (pixel x/y)',
151 coords_ref_factor = pexConfig.Field(
154 doc=
'Multiplicative factor for reference catalog coordinates.'
155 'If coords_spherical is true, this must be the number of degrees per unit increment of '
156 'column_ref_coord[12]. Otherwise, it must convert the coordinate to the same units'
157 ' as the target coordinates.',
159 coords_target_factor = pexConfig.Field(
162 doc=
'Multiplicative factor for target catalog coordinates.'
163 'If coords_spherical is true, this must be the number of degrees per unit increment of '
164 'column_target_coord[12]. Otherwise, it must convert the coordinate to the same units'
165 ' as the reference coordinates.',
167 coords_ref_to_convert = pexConfig.DictField(
172 dictCheck=
lambda x: len(x) == 2,
173 doc=
'Dict mapping sky coordinate columns to be converted to pixel columns',
175 mag_zeropoint_ref = pexConfig.Field(
178 doc=
'Magnitude zeropoint for reference catalog.',
183 catalog_ref: pd.DataFrame,
184 catalog_target: pd.DataFrame,
185 select_ref: np.array =
None,
186 select_target: np.array =
None,
187 radec_to_xy_func: Callable =
None,
188 return_converted_columns: bool =
False,
191 """Format matched catalogs that may require coordinate conversions.
195 catalog_ref : `pandas.DataFrame`
196 A reference catalog for comparison to `catalog_target`.
197 catalog_target : `pandas.DataFrame`
198 A target catalog
with measurements
for comparison to `catalog_ref`.
199 select_ref : `numpy.ndarray`, (Nref,)
200 A boolean array of len `catalog_ref`,
True for valid match candidates.
201 select_target : `numpy.ndarray`, (Ntarget,)
202 A boolean array of len `catalog_target`,
True for valid match candidates.
203 radec_to_xy_func : `typing.Callable`
204 Function taking equal-length ra, dec arrays
and returning an ndarray of
205 - ``x``: current parameter (`float`).
206 - ``extra_args``: additional arguments (`dict`).
207 return_converted_columns : `bool`
208 Whether to
return converted columns
in the `coord1`
and `coord2`
209 attributes, rather than keep the original values.
214 compcat_ref, compcat_target : `ComparableCatalog`
215 Comparable catalogs corresponding to the input reference
and target.
219 if convert_ref
and not callable(radec_to_xy_func):
220 raise TypeError(
'radec_to_xy_func must be callable if converting ref coords')
223 extras_ref, extras_target = (
224 CatalogExtras(catalog, select=select, coordinate_factor=coord_factor)
225 for catalog, select, coord_factor
in zip(
226 (catalog_ref, catalog_target),
227 (select_ref, select_target),
235 for catalog, extras, (column1, column2), convert
in (
240 _mul_column(catalog[column], extras.coordinate_factor)
241 for column
in (column1, column2)
246 coord = np.array([xy[idx_coord]
for xy
in xy_ref])
247 catalog[column_out] = coord
248 if convert_ref
and return_converted_columns:
250 coord1, coord2 = catalog[column1], catalog[column2]
251 if isinstance(coord1, pd.Series):
252 coord1 = coord1.values
253 if isinstance(coord2, pd.Series):
254 coord2 = coord2.values
257 catalog=catalog, column_coord1=column1, column_coord2=column2,
258 coord1=coord1, coord2=coord2, extras=extras,
261 return tuple(compcats)
265 """Configuration for the MatchProbabilistic matcher.
267 column_ref_order = pexConfig.Field(
271 doc=
"Name of column in reference catalog specifying order for matching."
272 " Derived from columns_ref_flux if not set.",
286 columns_all.extend(columns)
288 return set(columns_all)
303 columns_all.extend(columns)
304 return set(columns_all)
306 columns_ref_copy = pexConfig.ListField(
309 listCheck=
lambda x: len(set(x)) == len(x),
311 doc=
'Reference table columns to copy unchanged into both match tables',
313 columns_ref_flux = pexConfig.ListField(
316 listCheck=
lambda x: len(set(x)) == len(x),
318 doc=
"List of reference flux columns to nansum total magnitudes from if column_order is None",
320 columns_ref_meas = pexConfig.ListField(
322 doc=
'The reference table columns to compute match likelihoods from '
323 '(usually centroids and fluxes/magnitudes)',
325 columns_target_copy = pexConfig.ListField(
328 listCheck=
lambda x: len(set(x)) == len(x),
330 doc=
'Target table columns to copy unchanged into both match tables',
332 columns_target_meas = pexConfig.ListField(
334 doc=
'Target table columns with measurements corresponding to columns_ref_meas',
336 columns_target_err = pexConfig.ListField(
338 doc=
'Target table columns with standard errors (sigma) corresponding to columns_ref_meas',
340 columns_target_select_true = pexConfig.ListField(
342 default=(
'detect_isPrimary',),
343 doc=
'Target table columns to require to be True for selecting sources',
345 columns_target_select_false = pexConfig.ListField(
347 default=(
'merge_peak_sky',),
348 doc=
'Target table columns to require to be False for selecting sources',
350 coord_format = pexConfig.ConfigField(
351 dtype=ConvertCatalogCoordinatesConfig,
352 doc=
"Configuration for coordinate conversion",
354 mag_brightest_ref = pexConfig.Field(
357 doc=
'Bright magnitude cutoff for selecting reference sources to match.'
358 ' Ignored if column_ref_order is None.'
360 mag_faintest_ref = pexConfig.Field(
363 doc=
'Faint magnitude cutoff for selecting reference sources to match.'
364 ' Ignored if column_ref_order is None.'
366 match_dist_max = pexConfig.Field(
369 doc=
'Maximum match distance. Units must be arcseconds if coords_spherical, '
370 'or else match those of column_*_coord[12] multiplied by coords_*_factor.',
372 match_n_max = pexConfig.Field(
376 doc=
'Maximum number of spatial matches to consider (in ascending distance order).',
378 match_n_finite_min = pexConfig.Field(
382 doc=
'Minimum number of columns with a finite value to measure match likelihood',
384 order_ascending = pexConfig.Field(
388 doc=
'Whether to order reference match candidates in ascending order of column_ref_order '
389 '(should be False if the column is a flux and True if it is a magnitude.',
396 elif dtype == np.signedinteger:
398 elif dtype == np.unsignedinteger:
404 """A probabilistic, greedy catalog matcher.
408 config: `MatchProbabilisticConfig`
409 A configuration instance.
411 config: MatchProbabilisticConfig
415 config: MatchProbabilisticConfig,
421 catalog_ref: pd.DataFrame,
422 catalog_target: pd.DataFrame,
423 select_ref: np.array =
None,
424 select_target: np.array =
None,
425 logger: logging.Logger =
None,
426 logging_n_rows: int =
None,
433 catalog_ref : `pandas.DataFrame`
434 A reference catalog to match in order of a given column (i.e. greedily).
435 catalog_target : `pandas.DataFrame`
436 A target catalog
for matching sources
from `catalog_ref`. Must contain measurements
with errors.
437 select_ref : `numpy.array`
438 A boolean array of the same length
as `catalog_ref` selecting the sources that can be matched.
439 select_target : `numpy.array`
440 A boolean array of the same length
as `catalog_target` selecting the sources that can be matched.
441 logger : `logging.Logger`
442 A Logger
for logging.
443 logging_n_rows : `int`
444 The number of sources to match before printing a log message.
446 Additional keyword arguments to
pass to `format_catalogs`.
450 catalog_out_ref : `pandas.DataFrame`
451 A catalog of identical length to `catalog_ref`, containing match information
for rows selected by
452 `select_ref` (including the matching row index
in `catalog_target`).
453 catalog_out_target : `pandas.DataFrame`
454 A catalog of identical length to `catalog_target`, containing the indices of matching rows
in
456 exceptions : `dict` [`int`, `Exception`]
457 A dictionary keyed by `catalog_target` row number of the first exception caught when matching.
460 logger = logger_default
472 ref, target = config.coord_format.format_catalogs(
473 catalog_ref=catalog_ref, catalog_target=catalog_target,
474 select_ref=select_ref, select_target=select_target,
484 catalog_ref.loc[ref.extras.select, config.column_ref_order]
485 if config.column_ref_order
is not None else
486 np.nansum(catalog_ref.loc[ref.extras.select, config.columns_ref_flux], axis=1)
488 order = np.argsort(column_order
if config.order_ascending
else -column_order)
490 n_ref_select = len(ref.extras.indices)
492 match_dist_max = config.match_dist_max
493 coords_spherical = config.coord_format.coords_spherical
495 match_dist_max = np.radians(match_dist_max / 3600.)
498 func_convert = _radec_to_xyz
if coords_spherical
else np.vstack
499 vec_ref, vec_target = (
500 func_convert(cat.coord1[cat.extras.select], cat.coord2[cat.extras.select])
501 for cat
in (ref, target)
505 logger.info(
'Generating cKDTree with match_n_max=%d', config.match_n_max)
506 tree_obj = cKDTree(vec_target)
508 scores, idxs_target_select = tree_obj.query(
510 distance_upper_bound=match_dist_max,
511 k=config.match_n_max,
514 n_target_select = len(target.extras.indices)
515 n_matches = np.sum(idxs_target_select != n_target_select, axis=1)
516 n_matched_max = np.sum(n_matches == config.match_n_max)
517 if n_matched_max > 0:
519 '%d/%d (%.2f%%) selected true objects have n_matches=n_match_max(%d)',
520 n_matched_max, n_ref_select, 100.*n_matched_max/n_ref_select, config.match_n_max
524 target_row_match = np.full(target.extras.n, np.nan, dtype=np.int64)
525 ref_candidate_match = np.zeros(ref.extras.n, dtype=bool)
526 ref_row_match = np.full(ref.extras.n, np.nan, dtype=np.int64)
527 ref_match_count = np.zeros(ref.extras.n, dtype=np.int32)
528 ref_match_meas_finite = np.zeros(ref.extras.n, dtype=np.int32)
529 ref_chisq = np.full(ref.extras.n, np.nan, dtype=float)
532 idx_orig_ref, idx_orig_target = (np.argwhere(cat.extras.select)
for cat
in (ref, target))
535 columns_convert = config.coord_format.coords_ref_to_convert
536 if columns_convert
is None:
538 data_ref = ref.catalog[
539 [columns_convert.get(column, column)
for column
in config.columns_ref_meas]
540 ].iloc[ref.extras.indices[order]]
541 data_target = target.catalog[config.columns_target_meas][target.extras.select]
542 errors_target = target.catalog[config.columns_target_err][target.extras.select]
546 matched_target = {n_target_select, }
548 t_begin = time.process_time()
550 logger.info(
'Matching n_indices=%d/%d', len(order), len(ref.catalog))
551 for index_n, index_row_select
in enumerate(order):
552 index_row = idx_orig_ref[index_row_select]
553 ref_candidate_match[index_row] =
True
554 found = idxs_target_select[index_row_select, :]
559 found = [x
for x
in found
if x
not in matched_target]
564 (data_target.iloc[found].values - data_ref.iloc[index_n].values)
565 / errors_target.iloc[found].values
567 finite = np.isfinite(chi)
568 n_finite = np.sum(finite, axis=1)
570 chisq_good = n_finite >= config.match_n_finite_min
571 if np.any(chisq_good):
573 chisq_sum = np.zeros(n_found, dtype=float)
574 chisq_sum[chisq_good] = np.nansum(chi[chisq_good, :] ** 2, axis=1)
575 idx_chisq_min = np.nanargmin(chisq_sum / n_finite)
576 ref_match_meas_finite[index_row] = n_finite[idx_chisq_min]
577 ref_match_count[index_row] = len(chisq_good)
578 ref_chisq[index_row] = chisq_sum[idx_chisq_min]
579 idx_match_select = found[idx_chisq_min]
580 row_target = target.extras.indices[idx_match_select]
581 ref_row_match[index_row] = row_target
583 target_row_match[row_target] = index_row
584 matched_target.add(idx_match_select)
585 except Exception
as error:
588 exceptions[index_row] = error
590 if logging_n_rows
and ((index_n + 1) % logging_n_rows == 0):
591 t_elapsed = time.process_time() - t_begin
593 'Processed %d/%d in %.2fs at sort value=%.3f',
594 index_n + 1, n_ref_select, t_elapsed, column_order[order[index_n]],
598 'match_candidate': ref_candidate_match,
599 'match_row': ref_row_match,
600 'match_count': ref_match_count,
601 'match_chisq': ref_chisq,
602 'match_n_chisq_finite': ref_match_meas_finite,
605 'match_candidate': target.extras.select
if target.extras.select
is not None else (
606 np.ones(target.extras.n, dtype=bool)),
607 'match_row': target_row_match,
610 for (columns, out_original, out_matched, in_original, in_matched, matches)
in (
612 self.
config.columns_ref_copy,
620 self.
config.columns_target_copy,
628 matched = matches >= 0
629 idx_matched = matches[matched]
631 for column
in columns:
632 values = in_original.catalog[column]
633 out_original[column] = values
634 dtype = in_original.catalog[column].dtype
636 types = list(set((type(x)
for x
in values)))
638 raise RuntimeError(f
'Column {column} dtype={dtype} has multiple types={types}')
641 column_match = np.full(in_matched.extras.n,
default_value(dtype), dtype=dtype)
642 column_match[matched] = in_original.catalog[column][idx_matched]
643 out_matched[f
'match_{column}'] = column_match
645 catalog_out_ref = pd.DataFrame(data_ref)
646 catalog_out_target = pd.DataFrame(data_target)
648 return catalog_out_ref, catalog_out_target, exceptions
def format_catalogs(self, pd.DataFrame catalog_ref, pd.DataFrame catalog_target, np.array select_ref=None, np.array select_target=None, Callable radec_to_xy_func=None, bool return_converted_columns=False, **kwargs)
Set[str] columns_in_target(self)
columns_target_select_false
columns_target_select_true
Set[str] columns_in_ref(self)
def match(self, pd.DataFrame catalog_ref, pd.DataFrame catalog_target, np.array select_ref=None, np.array select_target=None, logging.Logger logger=None, int logging_n_rows=None, **kwargs)
def __init__(self, MatchProbabilisticConfig config)