Coverage for python/lsst/analysis/tools/atools/diffMatched.py: 20%

246 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 11:04 +0000

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "ReferenceGalaxySelector", 

25 "ReferenceObjectSelector", 

26 "ReferenceStarSelector", 

27 "MatchedRefCoaddToolBase", 

28 "MatchedRefCoaddDiffTool", 

29 "MatchedRefCoaddDiffColorTool", 

30 "MatchedRefCoaddDiffMagTool", 

31 "MatchedRefCoaddDiffPositionTool", 

32) 

33 

34import copy 

35from abc import abstractmethod 

36 

37from lsst.pex.config import DictField, Field 

38 

39from ..actions.vector import ( 

40 CalcBinnedStatsAction, 

41 ColorDiff, 

42 ColorError, 

43 ConstantValue, 

44 DivideVector, 

45 DownselectVector, 

46 LoadVector, 

47 MultiplyVector, 

48 SubtractVector, 

49) 

50from ..actions.vector.selectors import RangeSelector, ThresholdSelector, VectorSelector 

51from ..interfaces import KeyedData, Vector 

52from .genericBuild import MagnitudeXTool 

53from .genericMetricAction import StructMetricAction 

54from .genericPlotAction import StructPlotAction 

55from .genericProduce import MagnitudeScatterPlot 

56 

57 

58class ReferenceGalaxySelector(ThresholdSelector): 

59 """A selector that selects galaxies from a catalog with a 

60 boolean column identifying unresolved sources. 

61 """ 

62 

63 def __call__(self, data: KeyedData, **kwargs) -> Vector: 

64 result = super().__call__(data=data, **kwargs) 

65 if self.plotLabelKey: 

66 self._addValueToPlotInfo("true galaxies", **kwargs) 

67 return result 

68 

69 def setDefaults(self): 

70 super().setDefaults() 

71 self.op = "eq" 

72 self.threshold = 0 

73 self.plotLabelKey = "Selection: Galaxies" 

74 self.vectorKey = "refcat_is_pointsource" 

75 

76 

77class ReferenceObjectSelector(RangeSelector): 

78 """A selector that selects all objects from a catalog with a 

79 boolean column identifying unresolved sources. 

80 """ 

81 

82 def setDefaults(self): 

83 super().setDefaults() 

84 self.minimum = 0 

85 self.vectorKey = "refcat_is_pointsource" 

86 

87 

88class ReferenceStarSelector(ThresholdSelector): 

89 """A selector that selects stars from a catalog with a 

90 boolean column identifying unresolved sources. 

91 """ 

92 

93 def __call__(self, data: KeyedData, **kwargs) -> Vector: 

94 result = super().__call__(data=data, **kwargs) 

95 if self.plotLabelKey: 

96 self._addValueToPlotInfo("true stars", **kwargs) 

97 return result 

98 

99 def setDefaults(self): 

100 super().setDefaults() 

101 self.op = "eq" 

102 self.plotLabelKey = "Selection: Stars" 

103 self.threshold = 1 

104 self.vectorKey = "refcat_is_pointsource" 

105 

106 

107class MatchedRefCoaddToolBase(MagnitudeXTool): 

108 """Base tool for matched-to-reference metrics/plots on coadds. 

109 

110 Metrics/plots are expected to use the reference magnitude and 

111 require separate star/galaxy/all source selections. 

112 

113 Notes 

114 ----- 

115 The tool does not use a standard coadd flag selector, because 

116 it is expected that the matcher has been configured to select 

117 appropriate candidates (and stores a match_candidate column). 

118 """ 

119 

120 def setDefaults(self): 

121 super().setDefaults() 

122 self.mag_x = "ref_matched" 

123 self.process.buildActions.allSelector = ReferenceObjectSelector() 

124 self.process.buildActions.galaxySelector = ReferenceGalaxySelector() 

125 self.process.buildActions.starSelector = ReferenceStarSelector() 

126 

127 def finalize(self): 

128 super().finalize() 

129 self._set_flux_default("mag_x") 

130 

131 

132class MatchedRefCoaddDiffTool(MatchedRefCoaddToolBase): 

133 """Base tool for generic diffs between reference and measured values.""" 

134 

135 compute_chi = Field[bool]( 

136 default=False, 

137 doc="Whether to compute scaled flux residuals (chi) instead of magnitude differences", 

138 ) 

139 # These are optional because validate can be called before finalize 

140 # Validate should not fail in that case if it would otherwise succeed 

141 name_prefix = Field[str](doc="Prefix for metric key", default=None, optional=True) 

142 unit = Field[str](doc="Astropy unit of y-axis values", default=None, optional=True) 

143 

144 _mag_low_min: int = 15 

145 _mag_low_max: int = 27 

146 _mag_interval: int = 1 

147 

148 _names = {"stars": "star", "galaxies": "galaxy", "all": "all"} 

149 _types = ("unresolved", "resolved", "all") 

150 

151 def _set_actions(self, suffix=None): 

152 if suffix is None: 

153 suffix = "" 

154 

155 for name_lower, name_singular in self._names.items(): 

156 name = name_lower.capitalize() 

157 key = f"y{name}{suffix}" 

158 setattr( 

159 self.process.filterActions, 

160 key, 

161 DownselectVector( 

162 vectorKey=f"diff{suffix}", selector=VectorSelector(vectorKey=f"{name_singular}Selector") 

163 ), 

164 ) 

165 

166 for minimum in range(self._mag_low_min, self._mag_low_max + 1): 

167 setattr( 

168 self.process.calculateActions, 

169 f"{name_lower}{minimum}{suffix}", 

170 CalcBinnedStatsAction( 

171 key_vector=key, 

172 selector_range=RangeSelector( 

173 vectorKey=key, 

174 minimum=minimum, 

175 maximum=minimum + self._mag_interval, 

176 ), 

177 return_minmax=False, 

178 ), 

179 ) 

180 

181 def configureMetrics( 

182 self, 

183 unit: str | None = None, 

184 name_prefix: str | None = None, 

185 name_suffix: str = "_ref_mag{minimum}", 

186 attr_suffix: str | None = None, 

187 unit_select: str = "mag", 

188 ): 

189 """Configure metric actions and return units. 

190 

191 Parameters 

192 ---------- 

193 unit : `str` 

194 The (astropy) unit of the summary statistic metrics. 

195 name_prefix : `str` 

196 The prefix for the action (column) name. 

197 name_suffix : `str` 

198 The suffix for the action (column) name. 

199 attr_suffix : `str` 

200 The suffix for the attribute to assign the action to. 

201 unit_select : `str` 

202 The (astropy) unit of the selection (x-axis) column. Default "mag". 

203 

204 Returns 

205 ------- 

206 units : `dict` [`str`, `str`] 

207 A dict of the unit (value) for each metric name (key) 

208 """ 

209 if unit is None: 

210 unit = self.unit if self.unit is not None else "" 

211 if name_prefix is None: 

212 name_prefix = self.name_prefix if self.name_prefix is not None else "" 

213 if attr_suffix is None: 

214 attr_suffix = "" 

215 

216 if unit_select is None: 

217 unit_select = "mag" 

218 

219 key_flux = self.config_mag_x.key_flux 

220 

221 units = {} 

222 for name, name_class in zip(self._names, self._types): 

223 name_capital = name.capitalize() 

224 x_key = f"x{name_capital}" 

225 

226 for minimum in range(self._mag_low_min, self._mag_low_max + 1): 

227 action = getattr(self.process.calculateActions, f"{name}{minimum}{attr_suffix}") 

228 action.selector_range = RangeSelector( 

229 vectorKey=x_key, 

230 minimum=minimum, 

231 maximum=minimum + self._mag_interval, 

232 ) 

233 

234 action.name_prefix = name_prefix.format( 

235 key_flux=key_flux, 

236 name_class=name_class, 

237 ) 

238 if self.parameterizedBand: 

239 action.name_prefix = f"{{band}}_{action.name_prefix}" 

240 action.name_suffix = name_suffix.format(minimum=minimum) 

241 

242 units.update( 

243 { 

244 action.name_median: unit, 

245 action.name_sigmaMad: unit, 

246 action.name_count: "count", 

247 action.name_select_median: unit_select, 

248 } 

249 ) 

250 return units 

251 

252 @property 

253 def config_mag_y(self): 

254 """Return the y-axis magnitude config. 

255 

256 Although tools may not end up using any flux measures in metrics or 

257 plots, this should still be set to the flux measure that was matched 

258 or selected against in the catalog not used for the x-axis.""" 

259 mag_y = self.get_key_flux_y() 

260 if mag_y not in self.fluxes: 

261 raise KeyError(f"{mag_y=} not in {self.fluxes}; was finalize called?") 

262 # This is a logic error: it shouldn't be called before finalize 

263 assert mag_y in self.fluxes 

264 return self.fluxes[mag_y] 

265 

266 @abstractmethod 

267 def get_key_flux_y(self) -> str: 

268 """Return the key for the y-axis flux measure.""" 

269 raise NotImplementedError("subclasses must implement get_key_flux_y") 

270 

271 def setDefaults(self): 

272 super().setDefaults() 

273 self._set_actions() 

274 

275 

276class MatchedRefCoaddDiffColorTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot): 

277 """Tool for diffs between reference and measured coadd mags. 

278 

279 Notes 

280 ----- 

281 Since this tool requires at least two bands, it is essentially impossible 

282 to call on its own. 

283 """ 

284 

285 mag_y1 = Field[str](default="cmodel_err", doc="Flux field for first magnitude") 

286 mag_y2 = Field[str]( 

287 doc="Flux field for second magnitude (to subtract from first); default same as first", 

288 default=None, 

289 optional=True, 

290 ) 

291 bands = DictField[str, str]( 

292 doc="Bands for colors. ", 

293 default={"u": "g", "g": "r,i", "r": "i", "i": "z", "z": "y"}, 

294 ) 

295 band_separator = Field[str](default=",", doc="Separator for multiple bands") 

296 

297 def _split_bands(self, band_list: str): 

298 return band_list.split(self.band_separator) 

299 

300 def finalize(self): 

301 # Check if it has already been finalized 

302 if not hasattr(self.process.buildActions, "diff_0"): 

303 if self.mag_y2 is None: 

304 self.mag_y2 = self.mag_y1 

305 # Ensure mag_y1/2 are set before any plot finalizes 

306 # This may result in duplicate actions but these are just plain 

307 # column selectors so that's not a serious problem 

308 bands = {band1: self._split_bands(band2_list) for band1, band2_list in self.bands.items()} 

309 n_bands = 0 

310 

311 for band1, band2_list in bands.items(): 

312 for band2 in band2_list: 

313 mag_y1 = f"mag_y_{band1}" 

314 mag_y2 = f"mag_y_{band2}" 

315 mag_x1 = f"mag_x_{band1}" 

316 mag_x2 = f"mag_x_{band2}" 

317 self._set_flux_default(mag_y1, band=band1, name_mag=self.mag_y1) 

318 self._set_flux_default(mag_y2, band=band2, name_mag=self.mag_y2) 

319 self._set_flux_default(mag_x1, band=band1, name_mag=self.mag_x) 

320 self._set_flux_default(mag_x2, band=band2, name_mag=self.mag_x) 

321 n_bands += 1 

322 

323 self.suffixes_y_finalize = [f"_{idx}" for idx in range(n_bands)] 

324 super().finalize() 

325 

326 self.unit = "" if self.compute_chi else "mmag" 

327 subtype = "chi" if self.compute_chi else "diff" 

328 

329 metric_base = self.produce.metric 

330 plot_base = self.produce.plot 

331 

332 actions_metric = {} 

333 actions_plot = {} 

334 

335 config_mag_x = self.config_mag_x 

336 config_mag_y = self.config_mag_y 

337 name_short_x = config_mag_x.name_flux_short 

338 name_short_y = config_mag_y.name_flux_short 

339 

340 idx = 0 

341 for band1, band2_list in bands.items(): 

342 for band2 in band2_list: 

343 name_color = f"{band1}_minus_{band2}" 

344 # Keep this index-based to simplify finalize 

345 suffix_y = f"_{idx}" 

346 self._set_actions(suffix=suffix_y) 

347 metric = copy.copy(metric_base) 

348 self.name_prefix = ( 

349 f"photom_{name_short_y}_vs_{name_short_x}_color_{name_color}" 

350 f"_{subtype}_{{name_class}}_" 

351 ) 

352 metric.units = self.configureMetrics(attr_suffix=suffix_y) 

353 plot = copy.copy(plot_base) 

354 

355 plot.suffix_y = suffix_y 

356 plot.suffix_stat = suffix_y 

357 

358 mag_y1 = f"{self.mag_y1}_{band1}" 

359 mag_y2 = f"{self.mag_y2}_{band2}" 

360 mag_x1 = f"{self.mag_x}_{band1}" 

361 mag_x2 = f"{self.mag_x}_{band2}" 

362 

363 diff = ColorDiff( 

364 color1_flux1=getattr(self.process.buildActions, f"flux_{mag_y1}"), 

365 color1_flux2=getattr(self.process.buildActions, f"flux_{mag_y2}"), 

366 color2_flux1=getattr(self.process.buildActions, f"flux_{mag_x1}"), 

367 color2_flux2=getattr(self.process.buildActions, f"flux_{mag_x2}"), 

368 ) 

369 

370 if self.compute_chi: 

371 diff = DivideVector( 

372 actionA=diff, 

373 actionB=ColorError( 

374 flux_err1=DivideVector( 

375 actionA=getattr(self.process.buildActions, f"flux_err_{mag_y1}"), 

376 actionB=getattr(self.process.buildActions, f"flux_{mag_y1}"), 

377 ), 

378 flux_err2=DivideVector( 

379 actionA=getattr(self.process.buildActions, f"flux_err_{mag_y2}"), 

380 actionB=getattr(self.process.buildActions, f"flux_{mag_y2}"), 

381 ), 

382 ), 

383 ) 

384 setattr(self.process.buildActions, f"diff{plot.suffix_y}", diff) 

385 

386 label = f"({band1} - {band2}) ({config_mag_y.name_flux} - {config_mag_x.name_flux})" 

387 label = f"chi = ({label})/error" if self.compute_chi else f"{label} (mmag)" 

388 plot.yAxisLabel = label 

389 actions_metric[name_color] = metric 

390 actions_plot[name_color] = plot 

391 idx += 1 

392 action_metric = StructMetricAction() 

393 for name_action, action in actions_metric.items(): 

394 setattr(action_metric.actions, name_action, action) 

395 self.produce.metric = action_metric 

396 action_plot = StructPlotAction() 

397 for name_action, action in actions_plot.items(): 

398 setattr(action_plot.actions, name_action, action) 

399 self.produce.plot = action_plot 

400 

401 def get_key_flux_y(self) -> str: 

402 return self.mag_y1 

403 

404 def setDefaults(self): 

405 # skip MatchedRefCoaddDiffTool.setDefaults's _setActions call 

406 MatchedRefCoaddToolBase.setDefaults(self) 

407 MagnitudeScatterPlot.setDefaults(self) 

408 

409 def validate(self): 

410 super().validate() 

411 errors = [] 

412 for band1, band2_list in self.bands.items(): 

413 bands = self._split_bands(band2_list) 

414 if len(set(bands)) != len(bands): 

415 errors.append(f"value={band2_list} is not a set for key={band1}") 

416 if errors: 

417 raise ValueError("\n".join(errors)) 

418 

419 

420class MatchedRefCoaddDiffMagTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot): 

421 """Tool for diffs between reference and measured coadd mags.""" 

422 

423 mag_y = Field[str](default="cmodel_err", doc="Flux (magnitude) field to difference against ref") 

424 

425 def finalize(self): 

426 # Check if it has already been finalized 

427 if not hasattr(self.process.buildActions, "diff"): 

428 # Ensure mag_y is set before any plot finalizes 

429 self._set_flux_default("mag_y") 

430 super().finalize() 

431 name_short_x = self.config_mag_x.name_flux_short 

432 name_short_y = self.config_mag_y.name_flux_short 

433 

434 prefix_action = "flux" if self.compute_chi else "mag" 

435 action_diff = SubtractVector( 

436 actionA=getattr(self.process.buildActions, f"{prefix_action}_{self.mag_x}"), 

437 actionB=getattr(self.process.buildActions, f"{prefix_action}_{self.mag_y}"), 

438 ) 

439 

440 if self.compute_chi: 

441 key_err = f"flux_err_{self.mag_y}" 

442 action_err = ( 

443 getattr(self.process.buildActions, key_err) 

444 if hasattr(self.process.buildActions, key_err) 

445 else getattr(self.process.buildActions, f"flux_err_{self.mag_x}") 

446 ) 

447 self.process.buildActions.diff = DivideVector(actionA=action_diff, actionB=action_err) 

448 else: 

449 # set to mmag 

450 self.process.buildActions.diff = MultiplyVector( 

451 actionA=action_diff, 

452 actionB=ConstantValue(value=1000.0), 

453 ) 

454 if not self.produce.plot.yAxisLabel: 

455 label = f"{self.config_mag_y.name_flux} - {self.config_mag_x.name_flux}" 

456 self.produce.plot.yAxisLabel = ( 

457 f"chi = ({label})/error" if self.compute_chi else f"{label} (mmag)" 

458 ) 

459 if self.unit is None: 

460 self.unit = "" if self.compute_chi else "mmag" 

461 if self.name_prefix is None: 

462 subtype = "chi" if self.compute_chi else "diff" 

463 self.name_prefix = f"photom_{name_short_y}_vs_{name_short_x}_mag_{subtype}_{{name_class}}_" 

464 if not self.produce.metric.units: 

465 self.produce.metric.units = self.configureMetrics() 

466 

467 def get_key_flux_y(self) -> str: 

468 return self.mag_y 

469 

470 

471class MatchedRefCoaddDiffPositionTool(MatchedRefCoaddDiffTool, MagnitudeScatterPlot): 

472 """Tool for diffs between reference and measured coadd astrometry.""" 

473 

474 mag_sn = Field[str](default="cmodel_err", doc="Flux (magnitude) field to use for S/N binning on plot") 

475 scale_factor = Field[float]( 

476 doc="The factor to multiply positions by (i.e. the pixel scale if coordinates have pixel units)", 

477 default=200, 

478 ) 

479 coord_label = Field[str]( 

480 doc="The plot label for the astrometric variable (default coord_meas)", 

481 optional=True, 

482 default=None, 

483 ) 

484 coord_meas = Field[str]( 

485 doc="The key for measured values of the astrometric variable", 

486 optional=False, 

487 ) 

488 coord_ref = Field[str]( 

489 doc="The key for reference values of the astrometric variabler", 

490 optional=False, 

491 ) 

492 

493 def finalize(self): 

494 # Check if it has already been finalized 

495 if not hasattr(self.process.buildActions, "diff"): 

496 # Set before MagnitudeScatterPlot.finalize to undo PSF default. 

497 # Matched ref tables may not have PSF fluxes, or prefer CModel. 

498 self._set_flux_default("mag_sn") 

499 super().finalize() 

500 name = self.coord_label if self.coord_label else self.coord_meas 

501 self.process.buildActions.pos_meas = LoadVector(vectorKey=self.coord_meas) 

502 self.process.buildActions.pos_ref = LoadVector(vectorKey=self.coord_ref) 

503 name_short_x = self.config_mag_x.name_flux_short 

504 name_short_y = self.config_mag_y.name_flux_short 

505 

506 if self.compute_chi: 

507 self.process.buildActions.diff = DivideVector( 

508 actionA=SubtractVector( 

509 actionA=self.process.buildActions.pos_meas, 

510 actionB=self.process.buildActions.pos_ref, 

511 ), 

512 actionB=LoadVector(vectorKey=f"{self.process.buildActions.pos_meas.vectorKey}Err"), 

513 ) 

514 else: 

515 self.process.buildActions.diff = MultiplyVector( 

516 actionA=ConstantValue(value=self.scale_factor), 

517 actionB=SubtractVector( 

518 actionA=self.process.buildActions.pos_meas, 

519 actionB=self.process.buildActions.pos_ref, 

520 ), 

521 ) 

522 if self.unit is None: 

523 self.unit = "" if self.compute_chi else "mas" 

524 if self.name_prefix is None: 

525 subtype = "chi" if self.compute_chi else "diff" 

526 self.name_prefix = ( 

527 f"astrom_{name_short_y}_vs_{name_short_x}_{self.coord_meas}_coord_{subtype}" 

528 f"_{{name_class}}_" 

529 ) 

530 if not self.produce.metric.units: 

531 self.produce.metric.units = self.configureMetrics() 

532 if not self.produce.plot.yAxisLabel: 

533 label = f"({name_short_y} - {name_short_x})" 

534 self.produce.plot.yAxisLabel = ( 

535 f"chi = ({label} {name} coord)/error" 

536 if self.compute_chi 

537 else f"{label} {name} coord ({self.unit})" 

538 ) 

539 

540 def get_key_flux_y(self) -> str: 

541 return self.mag_sn