433 catalog_ref: pd.DataFrame,
434 catalog_target: pd.DataFrame,
435 select_ref: np.array =
None,
436 select_target: np.array =
None,
437 logger: logging.Logger =
None,
438 logging_n_rows: int =
None,
445 catalog_ref : `pandas.DataFrame`
446 A reference catalog to match in order of a given column (i.e. greedily).
447 catalog_target : `pandas.DataFrame`
448 A target catalog for matching sources from `catalog_ref`. Must contain measurements with errors.
449 select_ref : `numpy.array`
450 A boolean array of the same length as `catalog_ref` selecting the sources that can be matched.
451 select_target : `numpy.array`
452 A boolean array of the same length as `catalog_target` selecting the sources that can be matched.
453 logger : `logging.Logger`
454 A Logger for logging.
455 logging_n_rows : `int`
456 The number of sources to match before printing a log message.
458 Additional keyword arguments to pass to `format_catalogs`.
462 catalog_out_ref : `pandas.DataFrame`
463 A catalog of identical length to `catalog_ref`, containing match information for rows selected by
464 `select_ref` (including the matching row index in `catalog_target`).
465 catalog_out_target : `pandas.DataFrame`
466 A catalog of identical length to `catalog_target`, containing the indices of matching rows in
468 exceptions : `dict` [`int`, `Exception`]
469 A dictionary keyed by `catalog_target` row number of the first exception caught when matching.
472 logger = logger_default
484 ref, target = config.coord_format.format_catalogs(
485 catalog_ref=catalog_ref, catalog_target=catalog_target,
486 select_ref=select_ref, select_target=select_target,
496 catalog_ref.loc[ref.extras.select, config.column_ref_order]
497 if config.column_ref_order
is not None else
498 np.nansum(catalog_ref.loc[ref.extras.select, config.columns_ref_flux], axis=1)
500 order = np.argsort(column_order
if config.order_ascending
else -column_order)
502 n_ref_select = len(ref.extras.indices)
504 match_dist_max = config.match_dist_max
505 coords_spherical = config.coord_format.coords_spherical
507 match_dist_max = np.radians(match_dist_max / 3600.)
510 func_convert = _radec_to_xyz
if coords_spherical
else np.vstack
511 vec_ref, vec_target = (
512 func_convert(cat.coord1[cat.extras.select], cat.coord2[cat.extras.select])
513 for cat
in (ref, target)
517 logger.info(
'Generating cKDTree with match_n_max=%d', config.match_n_max)
518 tree_obj = cKDTree(vec_target)
520 scores, idxs_target_select = tree_obj.query(
522 distance_upper_bound=match_dist_max,
523 k=config.match_n_max,
526 n_target_select = len(target.extras.indices)
527 n_matches = np.sum(idxs_target_select != n_target_select, axis=1)
528 n_matched_max = np.sum(n_matches == config.match_n_max)
529 if n_matched_max > 0:
531 '%d/%d (%.2f%%) selected true objects have n_matches=n_match_max(%d)',
532 n_matched_max, n_ref_select, 100.*n_matched_max/n_ref_select, config.match_n_max
536 target_row_match = np.full(target.extras.n, np.nan, dtype=np.int64)
537 ref_candidate_match = np.zeros(ref.extras.n, dtype=bool)
538 ref_row_match = np.full(ref.extras.n, np.nan, dtype=np.int64)
539 ref_match_count = np.zeros(ref.extras.n, dtype=np.int32)
540 ref_match_meas_finite = np.zeros(ref.extras.n, dtype=np.int32)
541 ref_chisq = np.full(ref.extras.n, np.nan, dtype=float)
544 idx_orig_ref, idx_orig_target = (np.argwhere(cat.extras.select)
for cat
in (ref, target))
547 columns_convert = config.coord_format.coords_ref_to_convert
548 if columns_convert
is None:
550 data_ref = ref.catalog[
551 [columns_convert.get(column, column)
for column
in config.columns_ref_meas]
552 ].iloc[ref.extras.indices[order]]
553 data_target = target.catalog[config.columns_target_meas][target.extras.select]
554 errors_target = target.catalog[config.columns_target_err][target.extras.select]
558 matched_target = {n_target_select, }
560 t_begin = time.process_time()
562 logger.info(
'Matching n_indices=%d/%d', len(order), len(ref.catalog))
563 for index_n, index_row_select
in enumerate(order):
564 index_row = idx_orig_ref[index_row_select]
565 ref_candidate_match[index_row] =
True
566 found = idxs_target_select[index_row_select, :]
571 found = [x
for x
in found
if x
not in matched_target]
576 (data_target.iloc[found].values - data_ref.iloc[index_n].values)
577 / errors_target.iloc[found].values
579 finite = np.isfinite(chi)
580 n_finite = np.sum(finite, axis=1)
582 chisq_good = n_finite >= config.match_n_finite_min
583 if np.any(chisq_good):
585 chisq_sum = np.zeros(n_found, dtype=float)
586 chisq_sum[chisq_good] = np.nansum(chi[chisq_good, :] ** 2, axis=1)
587 idx_chisq_min = np.nanargmin(chisq_sum / n_finite)
588 ref_match_meas_finite[index_row] = n_finite[idx_chisq_min]
589 ref_match_count[index_row] = len(chisq_good)
590 ref_chisq[index_row] = chisq_sum[idx_chisq_min]
591 idx_match_select = found[idx_chisq_min]
592 row_target = target.extras.indices[idx_match_select]
593 ref_row_match[index_row] = row_target
595 target_row_match[row_target] = index_row
596 matched_target.add(idx_match_select)
597 except Exception
as error:
600 exceptions[index_row] = error
602 if logging_n_rows
and ((index_n + 1) % logging_n_rows == 0):
603 t_elapsed = time.process_time() - t_begin
605 'Processed %d/%d in %.2fs at sort value=%.3f',
606 index_n + 1, n_ref_select, t_elapsed, column_order[order[index_n]],
610 'match_candidate': ref_candidate_match,
611 'match_row': ref_row_match,
612 'match_count': ref_match_count,
613 'match_chisq': ref_chisq,
614 'match_n_chisq_finite': ref_match_meas_finite,
617 'match_candidate': target.extras.select
if target.extras.select
is not None else (
618 np.ones(target.extras.n, dtype=bool)),
619 'match_row': target_row_match,
622 for (columns, out_original, out_matched, in_original, in_matched, matches)
in (
640 matched = matches >= 0
641 idx_matched = matches[matched]
643 for column
in columns:
644 values = in_original.catalog[column]
645 out_original[column] = values
646 dtype = in_original.catalog[column].dtype
650 types = list(set((type(x)
for x
in values)))
652 raise RuntimeError(f
'Column {column} dtype={dtype} has multiple types={types}')
659 dtype = f
'<U{max(len(x) for x in values)}'
661 column_match = np.full(in_matched.extras.n, value_fill, dtype=dtype)
662 column_match[matched] = in_original.catalog[column][idx_matched]
663 out_matched[f
'match_{column}'] = column_match
665 catalog_out_ref = pd.DataFrame(data_ref)
666 catalog_out_target = pd.DataFrame(data_target)
668 return catalog_out_ref, catalog_out_target, exceptions