Coverage for python/lsst/meas/astrom/matcher_probabilistic.py: 27%

205 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-18 03:00 -0700

1# This file is part of meas_astrom. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ['ConvertCatalogCoordinatesConfig', 'MatchProbabilisticConfig', 'MatcherProbabilistic'] 

23 

24import lsst.pex.config as pexConfig 

25 

26from dataclasses import dataclass 

27import logging 

28import numpy as np 

29import pandas as pd 

30from scipy.spatial import cKDTree 

31import time 

32from typing import Callable, Set 

33 

34logger_default = logging.getLogger(__name__) 

35 

36 

37def _mul_column(column: np.array, value: float): 

38 if value is not None and value != 1: 

39 column *= value 

40 return column 

41 

42 

43def _radec_to_xyz(ra, dec): 

44 """Convert input ra/dec coordinates to spherical unit vectors. 

45 

46 Parameters 

47 ---------- 

48 ra, dec: `numpy.ndarray` 

49 Arrays of right ascension/declination in degrees. 

50 

51 Returns 

52 ------- 

53 vectors : `numpy.ndarray`, (N, 3) 

54 Output unit vectors. 

55 """ 

56 if ra.size != dec.size: 

57 raise ValueError('ra and dec must be same size') 

58 ras = np.radians(ra) 

59 decs = np.radians(dec) 

60 vectors = np.empty((ras.size, 3)) 

61 

62 sin_dec = np.sin(np.pi / 2 - decs) 

63 vectors[:, 0] = sin_dec * np.cos(ras) 

64 vectors[:, 1] = sin_dec * np.sin(ras) 

65 vectors[:, 2] = np.cos(np.pi / 2 - decs) 

66 

67 return vectors 

68 

69 

70@dataclass 

71class CatalogExtras: 

72 """Store frequently-reference (meta)data relevant for matching a catalog. 

73 

74 Parameters 

75 ---------- 

76 catalog : `pandas.DataFrame` 

77 A pandas catalog to store extra information for. 

78 select : `numpy.array` 

79 A numpy boolean array of the same length as catalog to be used for 

80 target selection. 

81 """ 

82 n: int 

83 indices: np.array 

84 select: np.array 

85 

86 coordinate_factor: float = None 

87 

88 def __init__(self, catalog: pd.DataFrame, select: np.array = None, coordinate_factor: float = None): 

89 self.n = len(catalog) 

90 self.select = np.ones(self.n, dtype=bool) if select is None else select 

91 self.indices = np.flatnonzero(select) if select is not None else np.arange(self.n) 

92 self.coordinate_factor = coordinate_factor 

93 

94 

95@dataclass(frozen=True) 

96class ComparableCatalog: 

97 """A catalog with sources with coordinate columns in some standard format/units. 

98 

99 catalog : `pandas.DataFrame` 

100 A catalog with comparable coordinate columns. 

101 column_coord1 : `str` 

102 The first spatial coordinate column name. 

103 column_coord2 : `str` 

104 The second spatial coordinate column name. 

105 coord1 : `numpy.array` 

106 The first spatial coordinate values. 

107 coord2 : `numpy.array` 

108 The second spatial coordinate values. 

109 extras : `CatalogExtras` 

110 Extra cached (meta)data for the `catalog`. 

111 """ 

112 catalog: pd.DataFrame 

113 column_coord1: str 

114 column_coord2: str 

115 coord1: np.array 

116 coord2: np.array 

117 extras: CatalogExtras 

118 

119 

120class ConvertCatalogCoordinatesConfig(pexConfig.Config): 

121 """Configuration for the MatchProbabilistic matcher. 

122 """ 

123 column_ref_coord1 = pexConfig.Field( 

124 dtype=str, 

125 default='ra', 

126 doc='The reference table column for the first spatial coordinate (usually x or ra).', 

127 ) 

128 column_ref_coord2 = pexConfig.Field( 

129 dtype=str, 

130 default='dec', 

131 doc='The reference table column for the second spatial coordinate (usually y or dec).' 

132 'Units must match column_ref_coord1.', 

133 ) 

134 column_target_coord1 = pexConfig.Field( 

135 dtype=str, 

136 default='coord_ra', 

137 doc='The target table column for the first spatial coordinate (usually x or ra).' 

138 'Units must match column_ref_coord1.', 

139 ) 

140 column_target_coord2 = pexConfig.Field( 

141 dtype=str, 

142 default='coord_dec', 

143 doc='The target table column for the second spatial coordinate (usually y or dec).' 

144 'Units must match column_ref_coord2.', 

145 ) 

146 coords_spherical = pexConfig.Field( 

147 dtype=bool, 

148 default=True, 

149 doc='Whether column_*_coord[12] are spherical coordinates (ra/dec) or not (pixel x/y)', 

150 ) 

151 coords_ref_factor = pexConfig.Field( 

152 dtype=float, 

153 default=1.0, 

154 doc='Multiplicative factor for reference catalog coordinates.' 

155 'If coords_spherical is true, this must be the number of degrees per unit increment of ' 

156 'column_ref_coord[12]. Otherwise, it must convert the coordinate to the same units' 

157 ' as the target coordinates.', 

158 ) 

159 coords_target_factor = pexConfig.Field( 

160 dtype=float, 

161 default=1.0, 

162 doc='Multiplicative factor for target catalog coordinates.' 

163 'If coords_spherical is true, this must be the number of degrees per unit increment of ' 

164 'column_target_coord[12]. Otherwise, it must convert the coordinate to the same units' 

165 ' as the reference coordinates.', 

166 ) 

167 coords_ref_to_convert = pexConfig.DictField( 167 ↛ exitline 167 didn't jump to the function exit

168 default=None, 

169 keytype=str, 

170 itemtype=str, 

171 dictCheck=lambda x: len(x) == 2, 

172 doc='Dict mapping sky coordinate columns to be converted to pixel columns', 

173 ) 

174 mag_zeropoint_ref = pexConfig.Field( 

175 dtype=float, 

176 default=31.4, 

177 doc='Magnitude zeropoint for reference catalog.', 

178 ) 

179 

180 def format_catalogs( 

181 self, 

182 catalog_ref: pd.DataFrame, 

183 catalog_target: pd.DataFrame, 

184 select_ref: np.array = None, 

185 select_target: np.array = None, 

186 radec_to_xy_func: Callable = None, 

187 return_converted_columns: bool = False, 

188 **kwargs, 

189 ): 

190 """Format matched catalogs that may require coordinate conversions. 

191 

192 Parameters 

193 ---------- 

194 catalog_ref : `pandas.DataFrame` 

195 A reference catalog for comparison to `catalog_target`. 

196 catalog_target : `pandas.DataFrame` 

197 A target catalog with measurements for comparison to `catalog_ref`. 

198 select_ref : `numpy.ndarray`, (Nref,) 

199 A boolean array of len `catalog_ref`, True for valid match candidates. 

200 select_target : `numpy.ndarray`, (Ntarget,) 

201 A boolean array of len `catalog_target`, True for valid match candidates. 

202 radec_to_xy_func : `typing.Callable` 

203 Function taking equal-length ra, dec arrays and returning an ndarray of 

204 - ``x``: current parameter (`float`). 

205 - ``extra_args``: additional arguments (`dict`). 

206 return_converted_columns : `bool` 

207 Whether to return converted columns in the `coord1` and `coord2` 

208 attributes, rather than keep the original values. 

209 kwargs 

210 

211 Returns 

212 ------- 

213 compcat_ref, compcat_target : `ComparableCatalog` 

214 Comparable catalogs corresponding to the input reference and target. 

215 

216 """ 

217 convert_ref = self.coords_ref_to_convert 

218 if convert_ref and not callable(radec_to_xy_func): 

219 raise TypeError('radec_to_xy_func must be callable if converting ref coords') 

220 

221 # Set up objects with frequently-used attributes like selection bool array 

222 extras_ref, extras_target = ( 

223 CatalogExtras(catalog, select=select, coordinate_factor=coord_factor) 

224 for catalog, select, coord_factor in zip( 

225 (catalog_ref, catalog_target), 

226 (select_ref, select_target), 

227 (self.coords_ref_factor, self.coords_target_factor), 

228 ) 

229 ) 

230 

231 compcats = [] 

232 

233 # Retrieve coordinates and multiply them by scaling factors 

234 for catalog, extras, (column1, column2), convert in ( 

235 (catalog_ref, extras_ref, (self.column_ref_coord1, self.column_ref_coord2), convert_ref), 

236 (catalog_target, extras_target, (self.column_target_coord1, self.column_target_coord2), False), 

237 ): 

238 coord1, coord2 = ( 

239 _mul_column(catalog[column], extras.coordinate_factor) 

240 for column in (column1, column2) 

241 ) 

242 if convert: 

243 xy_ref = radec_to_xy_func(coord1, coord2, self.coords_ref_factor, **kwargs) 

244 for idx_coord, column_out in enumerate(self.coords_ref_to_convert.values()): 

245 coord = np.array([xy[idx_coord] for xy in xy_ref]) 

246 catalog[column_out] = coord 

247 if convert_ref and return_converted_columns: 

248 column1, column2 = self.coords_ref_to_convert.values() 

249 coord1, coord2 = catalog[column1], catalog[column2] 

250 if isinstance(coord1, pd.Series): 

251 coord1 = coord1.values 

252 if isinstance(coord2, pd.Series): 

253 coord2 = coord2.values 

254 

255 compcats.append(ComparableCatalog( 

256 catalog=catalog, column_coord1=column1, column_coord2=column2, 

257 coord1=coord1, coord2=coord2, extras=extras, 

258 )) 

259 

260 return tuple(compcats) 

261 

262 

263class MatchProbabilisticConfig(pexConfig.Config): 

264 """Configuration for the MatchProbabilistic matcher. 

265 """ 

266 column_ref_order = pexConfig.Field( 

267 dtype=str, 

268 default=None, 

269 optional=True, 

270 doc="Name of column in reference catalog specifying order for matching." 

271 " Derived from columns_ref_flux if not set.", 

272 ) 

273 

274 @property 

275 def columns_in_ref(self) -> Set[str]: 

276 columns_all = [ 

277 self.coord_format.column_ref_coord1, 

278 self.coord_format.column_ref_coord2, 

279 ] 

280 for columns in ( 

281 self.columns_ref_flux, 

282 self.columns_ref_meas, 

283 self.columns_ref_copy, 

284 ): 

285 columns_all.extend(columns) 

286 

287 return set(columns_all) 

288 

289 @property 

290 def columns_in_target(self) -> Set[str]: 

291 columns_all = [ 

292 self.coord_format.column_target_coord1, 

293 self.coord_format.column_target_coord2, 

294 ] 

295 for columns in ( 

296 self.columns_target_meas, 

297 self.columns_target_err, 

298 self.columns_target_select_false, 

299 self.columns_target_select_true, 

300 self.columns_target_copy, 

301 ): 

302 columns_all.extend(columns) 

303 return set(columns_all) 

304 

305 columns_ref_copy = pexConfig.ListField( 305 ↛ exitline 305 didn't jump to the function exit

306 dtype=str, 

307 default=[], 

308 listCheck=lambda x: len(set(x)) == len(x), 

309 optional=True, 

310 doc='Reference table columns to copy unchanged into both match tables', 

311 ) 

312 columns_ref_flux = pexConfig.ListField( 312 ↛ exitline 312 didn't jump to the function exit

313 dtype=str, 

314 default=[], 

315 listCheck=lambda x: len(set(x)) == len(x), 

316 optional=True, 

317 doc="List of reference flux columns to nansum total magnitudes from if column_order is None", 

318 ) 

319 columns_ref_meas = pexConfig.ListField( 

320 dtype=str, 

321 doc='The reference table columns to compute match likelihoods from ' 

322 '(usually centroids and fluxes/magnitudes)', 

323 ) 

324 columns_target_copy = pexConfig.ListField( 324 ↛ exitline 324 didn't jump to the function exit

325 dtype=str, 

326 default=[], 

327 listCheck=lambda x: len(set(x)) == len(x), 

328 optional=True, 

329 doc='Target table columns to copy unchanged into both match tables', 

330 ) 

331 columns_target_meas = pexConfig.ListField( 

332 dtype=str, 

333 doc='Target table columns with measurements corresponding to columns_ref_meas', 

334 ) 

335 columns_target_err = pexConfig.ListField( 

336 dtype=str, 

337 doc='Target table columns with standard errors (sigma) corresponding to columns_ref_meas', 

338 ) 

339 columns_target_select_true = pexConfig.ListField( 

340 dtype=str, 

341 default=('detect_isPrimary',), 

342 doc='Target table columns to require to be True for selecting sources', 

343 ) 

344 columns_target_select_false = pexConfig.ListField( 

345 dtype=str, 

346 default=('merge_peak_sky',), 

347 doc='Target table columns to require to be False for selecting sources', 

348 ) 

349 coord_format = pexConfig.ConfigField( 

350 dtype=ConvertCatalogCoordinatesConfig, 

351 doc="Configuration for coordinate conversion", 

352 ) 

353 mag_brightest_ref = pexConfig.Field( 

354 dtype=float, 

355 default=-np.inf, 

356 doc='Bright magnitude cutoff for selecting reference sources to match.' 

357 ' Ignored if column_ref_order is None.' 

358 ) 

359 mag_faintest_ref = pexConfig.Field( 

360 dtype=float, 

361 default=np.Inf, 

362 doc='Faint magnitude cutoff for selecting reference sources to match.' 

363 ' Ignored if column_ref_order is None.' 

364 ) 

365 match_dist_max = pexConfig.Field( 

366 dtype=float, 

367 default=0.5, 

368 doc='Maximum match distance. Units must be arcseconds if coords_spherical, ' 

369 'or else match those of column_*_coord[12] multiplied by coords_*_factor.', 

370 ) 

371 match_n_max = pexConfig.Field( 

372 dtype=int, 

373 default=10, 

374 optional=True, 

375 doc='Maximum number of spatial matches to consider (in ascending distance order).', 

376 ) 

377 match_n_finite_min = pexConfig.Field( 

378 dtype=int, 

379 default=3, 

380 optional=True, 

381 doc='Minimum number of columns with a finite value to measure match likelihood', 

382 ) 

383 order_ascending = pexConfig.Field( 

384 dtype=bool, 

385 default=False, 

386 optional=True, 

387 doc='Whether to order reference match candidates in ascending order of column_ref_order ' 

388 '(should be False if the column is a flux and True if it is a magnitude.', 

389 ) 

390 

391 

392def default_value(dtype): 

393 if dtype == str: 

394 return '' 

395 elif dtype == np.signedinteger: 

396 return np.Inf 

397 elif dtype == np.unsignedinteger: 

398 return -np.Inf 

399 return None 

400 

401 

402class MatcherProbabilistic: 

403 """A probabilistic, greedy catalog matcher. 

404 

405 Parameters 

406 ---------- 

407 config: `MatchProbabilisticConfig` 

408 A configuration instance. 

409 """ 

410 config: MatchProbabilisticConfig 

411 

412 def __init__( 

413 self, 

414 config: MatchProbabilisticConfig, 

415 ): 

416 self.config = config 

417 

418 def match( 

419 self, 

420 catalog_ref: pd.DataFrame, 

421 catalog_target: pd.DataFrame, 

422 select_ref: np.array = None, 

423 select_target: np.array = None, 

424 logger: logging.Logger = None, 

425 logging_n_rows: int = None, 

426 **kwargs 

427 ): 

428 """Match catalogs. 

429 

430 Parameters 

431 ---------- 

432 catalog_ref : `pandas.DataFrame` 

433 A reference catalog to match in order of a given column (i.e. greedily). 

434 catalog_target : `pandas.DataFrame` 

435 A target catalog for matching sources from `catalog_ref`. Must contain measurements with errors. 

436 select_ref : `numpy.array` 

437 A boolean array of the same length as `catalog_ref` selecting the sources that can be matched. 

438 select_target : `numpy.array` 

439 A boolean array of the same length as `catalog_target` selecting the sources that can be matched. 

440 logger : `logging.Logger` 

441 A Logger for logging. 

442 logging_n_rows : `int` 

443 The number of sources to match before printing a log message. 

444 kwargs 

445 Additional keyword arguments to pass to `format_catalogs`. 

446 

447 Returns 

448 ------- 

449 catalog_out_ref : `pandas.DataFrame` 

450 A catalog of identical length to `catalog_ref`, containing match information for rows selected by 

451 `select_ref` (including the matching row index in `catalog_target`). 

452 catalog_out_target : `pandas.DataFrame` 

453 A catalog of identical length to `catalog_target`, containing the indices of matching rows in 

454 `catalog_ref`. 

455 exceptions : `dict` [`int`, `Exception`] 

456 A dictionary keyed by `catalog_target` row number of the first exception caught when matching. 

457 """ 

458 if logger is None: 

459 logger = logger_default 

460 

461 config = self.config 

462 

463 # Transform any coordinates, if required 

464 # Note: The returned objects contain the original catalogs, as well as 

465 # transformed coordinates, and the selection of sources for matching. 

466 # These might be identical to the arrays passed as kwargs, but that 

467 # depends on config settings. 

468 # For the rest of this function, the selection arrays will be used, 

469 # but the indices of the original, unfiltered catalog will also be 

470 # output, so some further indexing steps are needed. 

471 ref, target = config.coord_format.format_catalogs( 

472 catalog_ref=catalog_ref, catalog_target=catalog_target, 

473 select_ref=select_ref, select_target=select_target, 

474 **kwargs 

475 ) 

476 

477 # If no order is specified, take nansum of all flux columns for a 'total flux' 

478 # Note: it won't actually be a total flux if bands overlap significantly 

479 # (or it might define a filter with >100% efficiency 

480 # Also, this is done on the original dataframe as it's harder to accomplish 

481 # just with a recarray 

482 column_order = ( 

483 catalog_ref.loc[ref.extras.select, config.column_ref_order] 

484 if config.column_ref_order is not None else 

485 np.nansum(catalog_ref.loc[ref.extras.select, config.columns_ref_flux], axis=1) 

486 ) 

487 order = np.argsort(column_order if config.order_ascending else -column_order) 

488 

489 n_ref_select = len(ref.extras.indices) 

490 

491 match_dist_max = config.match_dist_max 

492 coords_spherical = config.coord_format.coords_spherical 

493 if coords_spherical: 

494 match_dist_max = np.radians(match_dist_max / 3600.) 

495 

496 # Convert ra/dec sky coordinates to spherical vectors for accurate distances 

497 func_convert = _radec_to_xyz if coords_spherical else np.vstack 

498 vec_ref, vec_target = ( 

499 func_convert(cat.coord1[cat.extras.select], cat.coord2[cat.extras.select]) 

500 for cat in (ref, target) 

501 ) 

502 

503 # Generate K-d tree to compute distances 

504 logger.info('Generating cKDTree with match_n_max=%d', config.match_n_max) 

505 tree_obj = cKDTree(vec_target) 

506 

507 scores, idxs_target_select = tree_obj.query( 

508 vec_ref, 

509 distance_upper_bound=match_dist_max, 

510 k=config.match_n_max, 

511 ) 

512 

513 n_target_select = len(target.extras.indices) 

514 n_matches = np.sum(idxs_target_select != n_target_select, axis=1) 

515 n_matched_max = np.sum(n_matches == config.match_n_max) 

516 if n_matched_max > 0: 

517 logger.warning( 

518 '%d/%d (%.2f%%) selected true objects have n_matches=n_match_max(%d)', 

519 n_matched_max, n_ref_select, 100.*n_matched_max/n_ref_select, config.match_n_max 

520 ) 

521 

522 # Pre-allocate outputs 

523 target_row_match = np.full(target.extras.n, np.nan, dtype=np.int64) 

524 ref_candidate_match = np.zeros(ref.extras.n, dtype=bool) 

525 ref_row_match = np.full(ref.extras.n, np.nan, dtype=np.int64) 

526 ref_match_count = np.zeros(ref.extras.n, dtype=np.int32) 

527 ref_match_meas_finite = np.zeros(ref.extras.n, dtype=np.int32) 

528 ref_chisq = np.full(ref.extras.n, np.nan, dtype=float) 

529 

530 # Need the original reference row indices for output 

531 idx_orig_ref, idx_orig_target = (np.argwhere(cat.extras.select) for cat in (ref, target)) 

532 

533 # Retrieve required columns, including any converted ones (default to original column name) 

534 columns_convert = config.coord_format.coords_ref_to_convert 

535 if columns_convert is None: 

536 columns_convert = {} 

537 data_ref = ref.catalog[ 

538 [columns_convert.get(column, column) for column in config.columns_ref_meas] 

539 ].iloc[ref.extras.indices[order]] 

540 data_target = target.catalog[config.columns_target_meas][target.extras.select] 

541 errors_target = target.catalog[config.columns_target_err][target.extras.select] 

542 

543 exceptions = {} 

544 # The kdTree uses len(inputs) as a sentinel value for no match 

545 matched_target = {n_target_select, } 

546 

547 t_begin = time.process_time() 

548 

549 logger.info('Matching n_indices=%d/%d', len(order), len(ref.catalog)) 

550 for index_n, index_row_select in enumerate(order): 

551 index_row = idx_orig_ref[index_row_select] 

552 ref_candidate_match[index_row] = True 

553 found = idxs_target_select[index_row_select, :] 

554 # Select match candidates from nearby sources not already matched 

555 # Note: set lookup is apparently fast enough that this is a few percent faster than: 

556 # found = [x for x in found[found != n_target_select] if x not in matched_target] 

557 # ... at least for ~1M sources 

558 found = [x for x in found if x not in matched_target] 

559 n_found = len(found) 

560 if n_found > 0: 

561 # This is an ndarray of n_found rows x len(data_ref/target) columns 

562 chi = ( 

563 (data_target.iloc[found].values - data_ref.iloc[index_n].values) 

564 / errors_target.iloc[found].values 

565 ) 

566 finite = np.isfinite(chi) 

567 n_finite = np.sum(finite, axis=1) 

568 # Require some number of finite chi_sq to match 

569 chisq_good = n_finite >= config.match_n_finite_min 

570 if np.any(chisq_good): 

571 try: 

572 chisq_sum = np.zeros(n_found, dtype=float) 

573 chisq_sum[chisq_good] = np.nansum(chi[chisq_good, :] ** 2, axis=1) 

574 idx_chisq_min = np.nanargmin(chisq_sum / n_finite) 

575 ref_match_meas_finite[index_row] = n_finite[idx_chisq_min] 

576 ref_match_count[index_row] = len(chisq_good) 

577 ref_chisq[index_row] = chisq_sum[idx_chisq_min] 

578 idx_match_select = found[idx_chisq_min] 

579 row_target = target.extras.indices[idx_match_select] 

580 ref_row_match[index_row] = row_target 

581 

582 target_row_match[row_target] = index_row 

583 matched_target.add(idx_match_select) 

584 except Exception as error: 

585 # Can't foresee any exceptions, but they shouldn't prevent 

586 # matching subsequent sources 

587 exceptions[index_row] = error 

588 

589 if logging_n_rows and ((index_n + 1) % logging_n_rows == 0): 

590 t_elapsed = time.process_time() - t_begin 

591 logger.info( 

592 'Processed %d/%d in %.2fs at sort value=%.3f', 

593 index_n + 1, n_ref_select, t_elapsed, column_order[order[index_n]], 

594 ) 

595 

596 data_ref = { 

597 'match_candidate': ref_candidate_match, 

598 'match_row': ref_row_match, 

599 'match_count': ref_match_count, 

600 'match_chisq': ref_chisq, 

601 'match_n_chisq_finite': ref_match_meas_finite, 

602 } 

603 data_target = { 

604 'match_candidate': target.extras.select if target.extras.select is not None else ( 

605 np.ones(target.extras.n, dtype=bool)), 

606 'match_row': target_row_match, 

607 } 

608 

609 for (columns, out_original, out_matched, in_original, in_matched, matches) in ( 

610 ( 

611 self.config.columns_ref_copy, 

612 data_ref, 

613 data_target, 

614 ref, 

615 target, 

616 target_row_match, 

617 ), 

618 ( 

619 self.config.columns_target_copy, 

620 data_target, 

621 data_ref, 

622 target, 

623 ref, 

624 ref_row_match, 

625 ), 

626 ): 

627 matched = matches >= 0 

628 idx_matched = matches[matched] 

629 

630 for column in columns: 

631 values = in_original.catalog[column] 

632 out_original[column] = values 

633 dtype = in_original.catalog[column].dtype 

634 if dtype == object: 

635 types = list(set((type(x) for x in values))) 

636 if len(types) != 1: 

637 raise RuntimeError(f'Column {column} dtype={dtype} has multiple types={types}') 

638 dtype = types[0] 

639 

640 column_match = np.full(in_matched.extras.n, default_value(dtype), dtype=dtype) 

641 column_match[matched] = in_original.catalog[column][idx_matched] 

642 out_matched[f'match_{column}'] = column_match 

643 

644 catalog_out_ref = pd.DataFrame(data_ref) 

645 catalog_out_target = pd.DataFrame(data_target) 

646 

647 return catalog_out_ref, catalog_out_target, exceptions