Coverage for python/lsst/meas/astrom/matcher_probabilistic.py: 26%

210 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-22 11:40 +0000

1# This file is part of meas_astrom. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ['ConvertCatalogCoordinatesConfig', 'MatchProbabilisticConfig', 'MatcherProbabilistic'] 

23 

24import lsst.pex.config as pexConfig 

25 

26from dataclasses import dataclass 

27import logging 

28import numpy as np 

29import pandas as pd 

30from scipy.spatial import cKDTree 

31import time 

32from typing import Callable, Set 

33 

34logger_default = logging.getLogger(__name__) 

35 

36 

37def _mul_column(column: np.array, value: float): 

38 if value is not None and value != 1: 

39 column *= value 

40 return column 

41 

42 

43def _radec_to_xyz(ra, dec): 

44 """Convert input ra/dec coordinates to spherical unit vectors. 

45 

46 Parameters 

47 ---------- 

48 ra, dec: `numpy.ndarray` 

49 Arrays of right ascension/declination in degrees. 

50 

51 Returns 

52 ------- 

53 vectors : `numpy.ndarray`, (N, 3) 

54 Output unit vectors. 

55 """ 

56 if ra.size != dec.size: 

57 raise ValueError('ra and dec must be same size') 

58 ras = np.radians(ra) 

59 decs = np.radians(dec) 

60 vectors = np.empty((ras.size, 3)) 

61 

62 sin_dec = np.sin(np.pi / 2 - decs) 

63 vectors[:, 0] = sin_dec * np.cos(ras) 

64 vectors[:, 1] = sin_dec * np.sin(ras) 

65 vectors[:, 2] = np.cos(np.pi / 2 - decs) 

66 

67 return vectors 

68 

69 

70@dataclass 

71class CatalogExtras: 

72 """Store frequently-reference (meta)data relevant for matching a catalog. 

73 

74 Parameters 

75 ---------- 

76 catalog : `pandas.DataFrame` 

77 A pandas catalog to store extra information for. 

78 select : `numpy.array` 

79 A numpy boolean array of the same length as catalog to be used for 

80 target selection. 

81 """ 

82 n: int 

83 indices: np.array 

84 select: np.array 

85 

86 coordinate_factor: float = None 

87 

88 def __init__(self, catalog: pd.DataFrame, select: np.array = None, coordinate_factor: float = None): 

89 self.n = len(catalog) 

90 self.select = np.ones(self.n, dtype=bool) if select is None else select 

91 self.indices = np.flatnonzero(select) if select is not None else np.arange(self.n) 

92 self.coordinate_factor = coordinate_factor 

93 

94 

95@dataclass(frozen=True) 

96class ComparableCatalog: 

97 """A catalog with sources with coordinate columns in some standard format/units. 

98 

99 catalog : `pandas.DataFrame` 

100 A catalog with comparable coordinate columns. 

101 column_coord1 : `str` 

102 The first spatial coordinate column name. 

103 column_coord2 : `str` 

104 The second spatial coordinate column name. 

105 coord1 : `numpy.array` 

106 The first spatial coordinate values. 

107 coord2 : `numpy.array` 

108 The second spatial coordinate values. 

109 extras : `CatalogExtras` 

110 Extra cached (meta)data for the `catalog`. 

111 """ 

112 catalog: pd.DataFrame 

113 column_coord1: str 

114 column_coord2: str 

115 coord1: np.array 

116 coord2: np.array 

117 extras: CatalogExtras 

118 

119 

120class ConvertCatalogCoordinatesConfig(pexConfig.Config): 

121 """Configuration for the MatchProbabilistic matcher. 

122 """ 

123 column_ref_coord1 = pexConfig.Field( 

124 dtype=str, 

125 default='ra', 

126 doc='The reference table column for the first spatial coordinate (usually x or ra).', 

127 ) 

128 column_ref_coord2 = pexConfig.Field( 

129 dtype=str, 

130 default='dec', 

131 doc='The reference table column for the second spatial coordinate (usually y or dec).' 

132 'Units must match column_ref_coord1.', 

133 ) 

134 column_target_coord1 = pexConfig.Field( 

135 dtype=str, 

136 default='coord_ra', 

137 doc='The target table column for the first spatial coordinate (usually x or ra).' 

138 'Units must match column_ref_coord1.', 

139 ) 

140 column_target_coord2 = pexConfig.Field( 

141 dtype=str, 

142 default='coord_dec', 

143 doc='The target table column for the second spatial coordinate (usually y or dec).' 

144 'Units must match column_ref_coord2.', 

145 ) 

146 coords_spherical = pexConfig.Field( 

147 dtype=bool, 

148 default=True, 

149 doc='Whether column_*_coord[12] are spherical coordinates (ra/dec) or not (pixel x/y)', 

150 ) 

151 coords_ref_factor = pexConfig.Field( 

152 dtype=float, 

153 default=1.0, 

154 doc='Multiplicative factor for reference catalog coordinates.' 

155 'If coords_spherical is true, this must be the number of degrees per unit increment of ' 

156 'column_ref_coord[12]. Otherwise, it must convert the coordinate to the same units' 

157 ' as the target coordinates.', 

158 ) 

159 coords_target_factor = pexConfig.Field( 

160 dtype=float, 

161 default=1.0, 

162 doc='Multiplicative factor for target catalog coordinates.' 

163 'If coords_spherical is true, this must be the number of degrees per unit increment of ' 

164 'column_target_coord[12]. Otherwise, it must convert the coordinate to the same units' 

165 ' as the reference coordinates.', 

166 ) 

167 coords_ref_to_convert = pexConfig.DictField( 167 ↛ exitline 167 didn't jump to the function exit

168 default=None, 

169 optional=True, 

170 keytype=str, 

171 itemtype=str, 

172 dictCheck=lambda x: len(x) == 2, 

173 doc='Dict mapping sky coordinate columns to be converted to pixel columns', 

174 ) 

175 mag_zeropoint_ref = pexConfig.Field( 

176 dtype=float, 

177 default=31.4, 

178 doc='Magnitude zeropoint for reference catalog.', 

179 ) 

180 

181 def format_catalogs( 

182 self, 

183 catalog_ref: pd.DataFrame, 

184 catalog_target: pd.DataFrame, 

185 select_ref: np.array = None, 

186 select_target: np.array = None, 

187 radec_to_xy_func: Callable = None, 

188 return_converted_columns: bool = False, 

189 **kwargs, 

190 ): 

191 """Format matched catalogs that may require coordinate conversions. 

192 

193 Parameters 

194 ---------- 

195 catalog_ref : `pandas.DataFrame` 

196 A reference catalog for comparison to `catalog_target`. 

197 catalog_target : `pandas.DataFrame` 

198 A target catalog with measurements for comparison to `catalog_ref`. 

199 select_ref : `numpy.ndarray`, (Nref,) 

200 A boolean array of len `catalog_ref`, True for valid match candidates. 

201 select_target : `numpy.ndarray`, (Ntarget,) 

202 A boolean array of len `catalog_target`, True for valid match candidates. 

203 radec_to_xy_func : `typing.Callable` 

204 Function taking equal-length ra, dec arrays and returning an ndarray of 

205 - ``x``: current parameter (`float`). 

206 - ``extra_args``: additional arguments (`dict`). 

207 return_converted_columns : `bool` 

208 Whether to return converted columns in the `coord1` and `coord2` 

209 attributes, rather than keep the original values. 

210 kwargs 

211 

212 Returns 

213 ------- 

214 compcat_ref, compcat_target : `ComparableCatalog` 

215 Comparable catalogs corresponding to the input reference and target. 

216 

217 """ 

218 convert_ref = self.coords_ref_to_convert 

219 if convert_ref and not callable(radec_to_xy_func): 

220 raise TypeError('radec_to_xy_func must be callable if converting ref coords') 

221 

222 # Set up objects with frequently-used attributes like selection bool array 

223 extras_ref, extras_target = ( 

224 CatalogExtras(catalog, select=select, coordinate_factor=coord_factor) 

225 for catalog, select, coord_factor in zip( 

226 (catalog_ref, catalog_target), 

227 (select_ref, select_target), 

228 (self.coords_ref_factor, self.coords_target_factor), 

229 ) 

230 ) 

231 

232 compcats = [] 

233 

234 # Retrieve coordinates and multiply them by scaling factors 

235 for catalog, extras, (column1, column2), convert in ( 

236 (catalog_ref, extras_ref, (self.column_ref_coord1, self.column_ref_coord2), convert_ref), 

237 (catalog_target, extras_target, (self.column_target_coord1, self.column_target_coord2), False), 

238 ): 

239 coord1, coord2 = ( 

240 _mul_column(catalog[column], extras.coordinate_factor) 

241 for column in (column1, column2) 

242 ) 

243 if convert: 

244 xy_ref = radec_to_xy_func(coord1, coord2, self.coords_ref_factor, **kwargs) 

245 for idx_coord, column_out in enumerate(self.coords_ref_to_convert.values()): 

246 coord = np.array([xy[idx_coord] for xy in xy_ref]) 

247 catalog[column_out] = coord 

248 if convert_ref and return_converted_columns: 

249 column1, column2 = self.coords_ref_to_convert.values() 

250 coord1, coord2 = catalog[column1], catalog[column2] 

251 if isinstance(coord1, pd.Series): 

252 coord1 = coord1.values 

253 if isinstance(coord2, pd.Series): 

254 coord2 = coord2.values 

255 

256 compcats.append(ComparableCatalog( 

257 catalog=catalog, column_coord1=column1, column_coord2=column2, 

258 coord1=coord1, coord2=coord2, extras=extras, 

259 )) 

260 

261 return tuple(compcats) 

262 

263 

264class MatchProbabilisticConfig(pexConfig.Config): 

265 """Configuration for the MatchProbabilistic matcher. 

266 """ 

267 column_ref_order = pexConfig.Field( 

268 dtype=str, 

269 default=None, 

270 optional=True, 

271 doc="Name of column in reference catalog specifying order for matching." 

272 " Derived from columns_ref_flux if not set.", 

273 ) 

274 

275 @property 

276 def columns_in_ref(self) -> Set[str]: 

277 columns_all = [ 

278 self.coord_format.column_ref_coord1, 

279 self.coord_format.column_ref_coord2, 

280 ] 

281 for columns in ( 

282 self.columns_ref_flux, 

283 self.columns_ref_meas, 

284 self.columns_ref_select_false, 

285 self.columns_ref_select_true, 

286 self.columns_ref_copy, 

287 ): 

288 columns_all.extend(columns) 

289 

290 return set(columns_all) 

291 

292 @property 

293 def columns_in_target(self) -> Set[str]: 

294 columns_all = [ 

295 self.coord_format.column_target_coord1, 

296 self.coord_format.column_target_coord2, 

297 ] 

298 for columns in ( 

299 self.columns_target_meas, 

300 self.columns_target_err, 

301 self.columns_target_select_false, 

302 self.columns_target_select_true, 

303 self.columns_target_copy, 

304 ): 

305 columns_all.extend(columns) 

306 return set(columns_all) 

307 

308 columns_ref_copy = pexConfig.ListField( 308 ↛ exitline 308 didn't jump to the function exit

309 dtype=str, 

310 default=[], 

311 listCheck=lambda x: len(set(x)) == len(x), 

312 optional=True, 

313 doc='Reference table columns to copy unchanged into both match tables', 

314 ) 

315 columns_ref_flux = pexConfig.ListField( 

316 dtype=str, 

317 default=[], 

318 optional=True, 

319 doc="List of reference flux columns to nansum total magnitudes from if column_order is None", 

320 ) 

321 columns_ref_meas = pexConfig.ListField( 

322 dtype=str, 

323 doc='The reference table columns to compute match likelihoods from ' 

324 '(usually centroids and fluxes/magnitudes)', 

325 ) 

326 columns_ref_select_true = pexConfig.ListField( 

327 dtype=str, 

328 default=tuple(), 

329 doc='Reference table columns to require to be True for selecting sources', 

330 ) 

331 columns_ref_select_false = pexConfig.ListField( 

332 dtype=str, 

333 default=tuple(), 

334 doc='Reference table columns to require to be False for selecting sources', 

335 ) 

336 columns_target_copy = pexConfig.ListField( 336 ↛ exitline 336 didn't jump to the function exit

337 dtype=str, 

338 default=[], 

339 listCheck=lambda x: len(set(x)) == len(x), 

340 optional=True, 

341 doc='Target table columns to copy unchanged into both match tables', 

342 ) 

343 columns_target_meas = pexConfig.ListField( 

344 dtype=str, 

345 doc='Target table columns with measurements corresponding to columns_ref_meas', 

346 ) 

347 columns_target_err = pexConfig.ListField( 

348 dtype=str, 

349 doc='Target table columns with standard errors (sigma) corresponding to columns_ref_meas', 

350 ) 

351 columns_target_select_true = pexConfig.ListField( 

352 dtype=str, 

353 default=('detect_isPrimary',), 

354 doc='Target table columns to require to be True for selecting sources', 

355 ) 

356 columns_target_select_false = pexConfig.ListField( 

357 dtype=str, 

358 default=('merge_peak_sky',), 

359 doc='Target table columns to require to be False for selecting sources', 

360 ) 

361 coord_format = pexConfig.ConfigField( 

362 dtype=ConvertCatalogCoordinatesConfig, 

363 doc="Configuration for coordinate conversion", 

364 ) 

365 mag_brightest_ref = pexConfig.Field( 

366 dtype=float, 

367 default=-np.inf, 

368 doc='Bright magnitude cutoff for selecting reference sources to match.' 

369 ' Ignored if column_ref_order is None.' 

370 ) 

371 mag_faintest_ref = pexConfig.Field( 

372 dtype=float, 

373 default=np.Inf, 

374 doc='Faint magnitude cutoff for selecting reference sources to match.' 

375 ' Ignored if column_ref_order is None.' 

376 ) 

377 match_dist_max = pexConfig.Field( 

378 dtype=float, 

379 default=0.5, 

380 doc='Maximum match distance. Units must be arcseconds if coords_spherical, ' 

381 'or else match those of column_*_coord[12] multiplied by coords_*_factor.', 

382 ) 

383 match_n_max = pexConfig.Field( 

384 dtype=int, 

385 default=10, 

386 optional=True, 

387 doc='Maximum number of spatial matches to consider (in ascending distance order).', 

388 ) 

389 match_n_finite_min = pexConfig.Field( 

390 dtype=int, 

391 default=3, 

392 optional=True, 

393 doc='Minimum number of columns with a finite value to measure match likelihood', 

394 ) 

395 order_ascending = pexConfig.Field( 

396 dtype=bool, 

397 default=False, 

398 optional=True, 

399 doc='Whether to order reference match candidates in ascending order of column_ref_order ' 

400 '(should be False if the column is a flux and True if it is a magnitude.', 

401 ) 

402 

403 

404def default_value(dtype): 

405 if dtype == str: 

406 return '' 

407 elif dtype == np.signedinteger: 

408 return np.Inf 

409 elif dtype == np.unsignedinteger: 

410 return -np.Inf 

411 return None 

412 

413 

414class MatcherProbabilistic: 

415 """A probabilistic, greedy catalog matcher. 

416 

417 Parameters 

418 ---------- 

419 config: `MatchProbabilisticConfig` 

420 A configuration instance. 

421 """ 

422 config: MatchProbabilisticConfig 

423 

424 def __init__( 

425 self, 

426 config: MatchProbabilisticConfig, 

427 ): 

428 self.config = config 

429 

430 def match( 

431 self, 

432 catalog_ref: pd.DataFrame, 

433 catalog_target: pd.DataFrame, 

434 select_ref: np.array = None, 

435 select_target: np.array = None, 

436 logger: logging.Logger = None, 

437 logging_n_rows: int = None, 

438 **kwargs 

439 ): 

440 """Match catalogs. 

441 

442 Parameters 

443 ---------- 

444 catalog_ref : `pandas.DataFrame` 

445 A reference catalog to match in order of a given column (i.e. greedily). 

446 catalog_target : `pandas.DataFrame` 

447 A target catalog for matching sources from `catalog_ref`. Must contain measurements with errors. 

448 select_ref : `numpy.array` 

449 A boolean array of the same length as `catalog_ref` selecting the sources that can be matched. 

450 select_target : `numpy.array` 

451 A boolean array of the same length as `catalog_target` selecting the sources that can be matched. 

452 logger : `logging.Logger` 

453 A Logger for logging. 

454 logging_n_rows : `int` 

455 The number of sources to match before printing a log message. 

456 kwargs 

457 Additional keyword arguments to pass to `format_catalogs`. 

458 

459 Returns 

460 ------- 

461 catalog_out_ref : `pandas.DataFrame` 

462 A catalog of identical length to `catalog_ref`, containing match information for rows selected by 

463 `select_ref` (including the matching row index in `catalog_target`). 

464 catalog_out_target : `pandas.DataFrame` 

465 A catalog of identical length to `catalog_target`, containing the indices of matching rows in 

466 `catalog_ref`. 

467 exceptions : `dict` [`int`, `Exception`] 

468 A dictionary keyed by `catalog_target` row number of the first exception caught when matching. 

469 """ 

470 if logger is None: 

471 logger = logger_default 

472 

473 config = self.config 

474 

475 # Transform any coordinates, if required 

476 # Note: The returned objects contain the original catalogs, as well as 

477 # transformed coordinates, and the selection of sources for matching. 

478 # These might be identical to the arrays passed as kwargs, but that 

479 # depends on config settings. 

480 # For the rest of this function, the selection arrays will be used, 

481 # but the indices of the original, unfiltered catalog will also be 

482 # output, so some further indexing steps are needed. 

483 ref, target = config.coord_format.format_catalogs( 

484 catalog_ref=catalog_ref, catalog_target=catalog_target, 

485 select_ref=select_ref, select_target=select_target, 

486 **kwargs 

487 ) 

488 

489 # If no order is specified, take nansum of all flux columns for a 'total flux' 

490 # Note: it won't actually be a total flux if bands overlap significantly 

491 # (or it might define a filter with >100% efficiency 

492 # Also, this is done on the original dataframe as it's harder to accomplish 

493 # just with a recarray 

494 column_order = ( 

495 catalog_ref.loc[ref.extras.select, config.column_ref_order] 

496 if config.column_ref_order is not None else 

497 np.nansum(catalog_ref.loc[ref.extras.select, config.columns_ref_flux], axis=1) 

498 ) 

499 order = np.argsort(column_order if config.order_ascending else -column_order) 

500 

501 n_ref_select = len(ref.extras.indices) 

502 

503 match_dist_max = config.match_dist_max 

504 coords_spherical = config.coord_format.coords_spherical 

505 if coords_spherical: 

506 match_dist_max = np.radians(match_dist_max / 3600.) 

507 

508 # Convert ra/dec sky coordinates to spherical vectors for accurate distances 

509 func_convert = _radec_to_xyz if coords_spherical else np.vstack 

510 vec_ref, vec_target = ( 

511 func_convert(cat.coord1[cat.extras.select], cat.coord2[cat.extras.select]) 

512 for cat in (ref, target) 

513 ) 

514 

515 # Generate K-d tree to compute distances 

516 logger.info('Generating cKDTree with match_n_max=%d', config.match_n_max) 

517 tree_obj = cKDTree(vec_target) 

518 

519 scores, idxs_target_select = tree_obj.query( 

520 vec_ref, 

521 distance_upper_bound=match_dist_max, 

522 k=config.match_n_max, 

523 ) 

524 

525 n_target_select = len(target.extras.indices) 

526 n_matches = np.sum(idxs_target_select != n_target_select, axis=1) 

527 n_matched_max = np.sum(n_matches == config.match_n_max) 

528 if n_matched_max > 0: 

529 logger.warning( 

530 '%d/%d (%.2f%%) selected true objects have n_matches=n_match_max(%d)', 

531 n_matched_max, n_ref_select, 100.*n_matched_max/n_ref_select, config.match_n_max 

532 ) 

533 

534 # Pre-allocate outputs 

535 target_row_match = np.full(target.extras.n, np.nan, dtype=np.int64) 

536 ref_candidate_match = np.zeros(ref.extras.n, dtype=bool) 

537 ref_row_match = np.full(ref.extras.n, np.nan, dtype=np.int64) 

538 ref_match_count = np.zeros(ref.extras.n, dtype=np.int32) 

539 ref_match_meas_finite = np.zeros(ref.extras.n, dtype=np.int32) 

540 ref_chisq = np.full(ref.extras.n, np.nan, dtype=float) 

541 

542 # Need the original reference row indices for output 

543 idx_orig_ref, idx_orig_target = (np.argwhere(cat.extras.select) for cat in (ref, target)) 

544 

545 # Retrieve required columns, including any converted ones (default to original column name) 

546 columns_convert = config.coord_format.coords_ref_to_convert 

547 if columns_convert is None: 

548 columns_convert = {} 

549 data_ref = ref.catalog[ 

550 [columns_convert.get(column, column) for column in config.columns_ref_meas] 

551 ].iloc[ref.extras.indices[order]] 

552 data_target = target.catalog[config.columns_target_meas][target.extras.select] 

553 errors_target = target.catalog[config.columns_target_err][target.extras.select] 

554 

555 exceptions = {} 

556 # The kdTree uses len(inputs) as a sentinel value for no match 

557 matched_target = {n_target_select, } 

558 

559 t_begin = time.process_time() 

560 

561 logger.info('Matching n_indices=%d/%d', len(order), len(ref.catalog)) 

562 for index_n, index_row_select in enumerate(order): 

563 index_row = idx_orig_ref[index_row_select] 

564 ref_candidate_match[index_row] = True 

565 found = idxs_target_select[index_row_select, :] 

566 # Select match candidates from nearby sources not already matched 

567 # Note: set lookup is apparently fast enough that this is a few percent faster than: 

568 # found = [x for x in found[found != n_target_select] if x not in matched_target] 

569 # ... at least for ~1M sources 

570 found = [x for x in found if x not in matched_target] 

571 n_found = len(found) 

572 if n_found > 0: 

573 # This is an ndarray of n_found rows x len(data_ref/target) columns 

574 chi = ( 

575 (data_target.iloc[found].values - data_ref.iloc[index_n].values) 

576 / errors_target.iloc[found].values 

577 ) 

578 finite = np.isfinite(chi) 

579 n_finite = np.sum(finite, axis=1) 

580 # Require some number of finite chi_sq to match 

581 chisq_good = n_finite >= config.match_n_finite_min 

582 if np.any(chisq_good): 

583 try: 

584 chisq_sum = np.zeros(n_found, dtype=float) 

585 chisq_sum[chisq_good] = np.nansum(chi[chisq_good, :] ** 2, axis=1) 

586 idx_chisq_min = np.nanargmin(chisq_sum / n_finite) 

587 ref_match_meas_finite[index_row] = n_finite[idx_chisq_min] 

588 ref_match_count[index_row] = len(chisq_good) 

589 ref_chisq[index_row] = chisq_sum[idx_chisq_min] 

590 idx_match_select = found[idx_chisq_min] 

591 row_target = target.extras.indices[idx_match_select] 

592 ref_row_match[index_row] = row_target 

593 

594 target_row_match[row_target] = index_row 

595 matched_target.add(idx_match_select) 

596 except Exception as error: 

597 # Can't foresee any exceptions, but they shouldn't prevent 

598 # matching subsequent sources 

599 exceptions[index_row] = error 

600 

601 if logging_n_rows and ((index_n + 1) % logging_n_rows == 0): 

602 t_elapsed = time.process_time() - t_begin 

603 logger.info( 

604 'Processed %d/%d in %.2fs at sort value=%.3f', 

605 index_n + 1, n_ref_select, t_elapsed, column_order[order[index_n]], 

606 ) 

607 

608 data_ref = { 

609 'match_candidate': ref_candidate_match, 

610 'match_row': ref_row_match, 

611 'match_count': ref_match_count, 

612 'match_chisq': ref_chisq, 

613 'match_n_chisq_finite': ref_match_meas_finite, 

614 } 

615 data_target = { 

616 'match_candidate': target.extras.select if target.extras.select is not None else ( 

617 np.ones(target.extras.n, dtype=bool)), 

618 'match_row': target_row_match, 

619 } 

620 

621 for (columns, out_original, out_matched, in_original, in_matched, matches) in ( 

622 ( 

623 self.config.columns_ref_copy, 

624 data_ref, 

625 data_target, 

626 ref, 

627 target, 

628 target_row_match, 

629 ), 

630 ( 

631 self.config.columns_target_copy, 

632 data_target, 

633 data_ref, 

634 target, 

635 ref, 

636 ref_row_match, 

637 ), 

638 ): 

639 matched = matches >= 0 

640 idx_matched = matches[matched] 

641 

642 for column in columns: 

643 values = in_original.catalog[column] 

644 out_original[column] = values 

645 dtype = in_original.catalog[column].dtype 

646 

647 # Pandas object columns can have mixed types - check for that 

648 if dtype == object: 

649 types = list(set((type(x) for x in values))) 

650 if len(types) != 1: 

651 raise RuntimeError(f'Column {column} dtype={dtype} has multiple types={types}') 

652 dtype = types[0] 

653 

654 value_fill = default_value(dtype) 

655 

656 # Without this, the dtype would be '<U1' for an empty Unicode string 

657 if dtype == str: 

658 dtype = f'<U{max(len(x) for x in values)}' 

659 

660 column_match = np.full(in_matched.extras.n, value_fill, dtype=dtype) 

661 column_match[matched] = in_original.catalog[column][idx_matched] 

662 out_matched[f'match_{column}'] = column_match 

663 

664 catalog_out_ref = pd.DataFrame(data_ref) 

665 catalog_out_target = pd.DataFrame(data_target) 

666 

667 return catalog_out_ref, catalog_out_target, exceptions