Coverage for python/lsst/pipe/tasks/calibrateImage.py: 24%

263 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-01 04:34 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections.abc 

23 

24import numpy as np 

25 

26import lsst.afw.table as afwTable 

27import lsst.afw.image as afwImage 

28import lsst.meas.algorithms 

29import lsst.meas.algorithms.installGaussianPsf 

30import lsst.meas.algorithms.measureApCorr 

31import lsst.meas.algorithms.setPrimaryFlags 

32import lsst.meas.base 

33import lsst.meas.astrom 

34import lsst.meas.deblender 

35import lsst.meas.extensions.shapeHSM 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38from lsst.pipe.base import connectionTypes 

39from lsst.utils.timer import timeMethod 

40 

41from . import measurePsf, repair, photoCal, computeExposureSummaryStats, snapCombine 

42 

43 

44class CalibrateImageConnections(pipeBase.PipelineTaskConnections, 

45 dimensions=("instrument", "visit", "detector")): 

46 

47 astrometry_ref_cat = connectionTypes.PrerequisiteInput( 

48 doc="Reference catalog to use for astrometric calibration.", 

49 name="gaia_dr3_20230707", 

50 storageClass="SimpleCatalog", 

51 dimensions=("skypix",), 

52 deferLoad=True, 

53 multiple=True, 

54 ) 

55 photometry_ref_cat = connectionTypes.PrerequisiteInput( 

56 doc="Reference catalog to use for photometric calibration.", 

57 name="ps1_pv3_3pi_20170110", 

58 storageClass="SimpleCatalog", 

59 dimensions=("skypix",), 

60 deferLoad=True, 

61 multiple=True 

62 ) 

63 

64 exposures = connectionTypes.Input( 

65 doc="Exposure (or two snaps) to be calibrated, and detected and measured on.", 

66 name="postISRCCD", 

67 storageClass="Exposure", 

68 multiple=True, # to handle 1 exposure or 2 snaps 

69 dimensions=["instrument", "exposure", "detector"], 

70 ) 

71 

72 # outputs 

73 initial_stars_schema = connectionTypes.InitOutput( 

74 doc="Schema of the output initial stars catalog.", 

75 name="initial_stars_schema", 

76 storageClass="SourceCatalog", 

77 ) 

78 

79 # TODO DM-38732: We want some kind of flag on Exposures/Catalogs to make 

80 # it obvious which components had failed to be computed/persisted. 

81 exposure = connectionTypes.Output( 

82 doc="Photometrically calibrated exposure with fitted calibrations and summary statistics.", 

83 name="initial_pvi", 

84 storageClass="ExposureF", 

85 dimensions=("instrument", "visit", "detector"), 

86 ) 

87 stars = connectionTypes.Output( 

88 doc="Catalog of unresolved sources detected on the calibrated exposure.", 

89 name="initial_stars_detector", 

90 storageClass="ArrowAstropy", 

91 dimensions=["instrument", "visit", "detector"], 

92 ) 

93 stars_footprints = connectionTypes.Output( 

94 doc="Catalog of unresolved sources detected on the calibrated exposure; " 

95 "includes source footprints.", 

96 name="initial_stars_footprints_detector", 

97 storageClass="SourceCatalog", 

98 dimensions=["instrument", "visit", "detector"], 

99 ) 

100 applied_photo_calib = connectionTypes.Output( 

101 doc="Photometric calibration that was applied to exposure.", 

102 name="initial_photoCalib_detector", 

103 storageClass="PhotoCalib", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 background = connectionTypes.Output( 

107 doc="Background models estimated during calibration task.", 

108 name="initial_pvi_background", 

109 storageClass="Background", 

110 dimensions=("instrument", "visit", "detector"), 

111 ) 

112 

113 # Optional outputs 

114 psf_stars_footprints = connectionTypes.Output( 

115 doc="Catalog of bright unresolved sources detected on the exposure used for PSF determination; " 

116 "includes source footprints.", 

117 name="initial_psf_stars_footprints_detector", 

118 storageClass="SourceCatalog", 

119 dimensions=["instrument", "visit", "detector"], 

120 ) 

121 psf_stars = connectionTypes.Output( 

122 doc="Catalog of bright unresolved sources detected on the exposure used for PSF determination.", 

123 name="initial_psf_stars_detector", 

124 storageClass="ArrowAstropy", 

125 dimensions=["instrument", "visit", "detector"], 

126 ) 

127 astrometry_matches = connectionTypes.Output( 

128 doc="Source to reference catalog matches from the astrometry solver.", 

129 name="initial_astrometry_match_detector", 

130 storageClass="Catalog", 

131 dimensions=("instrument", "visit", "detector"), 

132 ) 

133 photometry_matches = connectionTypes.Output( 

134 doc="Source to reference catalog matches from the photometry solver.", 

135 name="initial_photometry_match_detector", 

136 storageClass="Catalog", 

137 dimensions=("instrument", "visit", "detector"), 

138 ) 

139 

140 def __init__(self, *, config=None): 

141 super().__init__(config=config) 

142 if not config.optional_outputs: 

143 del self.psf_stars 

144 del self.psf_stars_footprints 

145 del self.astrometry_matches 

146 del self.photometry_matches 

147 

148 

149class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateImageConnections): 

150 optional_outputs = pexConfig.ListField( 

151 doc="Which optional outputs to save (as their connection name)?", 

152 dtype=str, 

153 # TODO: note somewhere to disable this for benchmarking, but should 

154 # we always have it on for production runs? 

155 default=["psf_stars", "psf_stars_footprints", "astrometry_matches", "photometry_matches"], 

156 optional=True 

157 ) 

158 

159 # To generate catalog ids consistently across subtasks. 

160 id_generator = lsst.meas.base.DetectorVisitIdGeneratorConfig.make_field() 

161 

162 snap_combine = pexConfig.ConfigurableField( 

163 target=snapCombine.SnapCombineTask, 

164 doc="Task to combine two snaps to make one exposure.", 

165 ) 

166 

167 # subtasks used during psf characterization 

168 install_simple_psf = pexConfig.ConfigurableField( 

169 target=lsst.meas.algorithms.installGaussianPsf.InstallGaussianPsfTask, 

170 doc="Task to install a simple PSF model into the input exposure to use " 

171 "when detecting bright sources for PSF estimation.", 

172 ) 

173 psf_repair = pexConfig.ConfigurableField( 

174 target=repair.RepairTask, 

175 doc="Task to repair cosmic rays on the exposure before PSF determination.", 

176 ) 

177 psf_subtract_background = pexConfig.ConfigurableField( 

178 target=lsst.meas.algorithms.SubtractBackgroundTask, 

179 doc="Task to perform intial background subtraction, before first detection pass.", 

180 ) 

181 psf_detection = pexConfig.ConfigurableField( 

182 target=lsst.meas.algorithms.SourceDetectionTask, 

183 doc="Task to detect sources for PSF determination." 

184 ) 

185 psf_source_measurement = pexConfig.ConfigurableField( 

186 target=lsst.meas.base.SingleFrameMeasurementTask, 

187 doc="Task to measure sources to be used for psf estimation." 

188 ) 

189 psf_measure_psf = pexConfig.ConfigurableField( 

190 target=measurePsf.MeasurePsfTask, 

191 doc="Task to measure the psf on bright sources." 

192 ) 

193 

194 # TODO DM-39203: we can remove aperture correction from this task once we are 

195 # using the shape-based star/galaxy code. 

196 measure_aperture_correction = pexConfig.ConfigurableField( 

197 target=lsst.meas.algorithms.measureApCorr.MeasureApCorrTask, 

198 doc="Task to compute the aperture correction from the bright stars." 

199 ) 

200 

201 # subtasks used during star measurement 

202 star_detection = pexConfig.ConfigurableField( 

203 target=lsst.meas.algorithms.SourceDetectionTask, 

204 doc="Task to detect stars to return in the output catalog." 

205 ) 

206 star_sky_sources = pexConfig.ConfigurableField( 

207 target=lsst.meas.algorithms.SkyObjectsTask, 

208 doc="Task to generate sky sources ('empty' regions where there are no detections).", 

209 ) 

210 star_deblend = pexConfig.ConfigurableField( 

211 target=lsst.meas.deblender.SourceDeblendTask, 

212 doc="Split blended sources into their components." 

213 ) 

214 star_measurement = pexConfig.ConfigurableField( 

215 target=lsst.meas.base.SingleFrameMeasurementTask, 

216 doc="Task to measure stars to return in the output catalog." 

217 ) 

218 star_apply_aperture_correction = pexConfig.ConfigurableField( 

219 target=lsst.meas.base.ApplyApCorrTask, 

220 doc="Task to apply aperture corrections to the selected stars." 

221 ) 

222 star_catalog_calculation = pexConfig.ConfigurableField( 

223 target=lsst.meas.base.CatalogCalculationTask, 

224 doc="Task to compute extendedness values on the star catalog, " 

225 "for the star selector to remove extended sources." 

226 ) 

227 star_set_primary_flags = pexConfig.ConfigurableField( 

228 target=lsst.meas.algorithms.setPrimaryFlags.SetPrimaryFlagsTask, 

229 doc="Task to add isPrimary to the catalog." 

230 ) 

231 star_selector = lsst.meas.algorithms.sourceSelectorRegistry.makeField( 

232 default="science", 

233 doc="Task to select isolated stars to use for calibration." 

234 ) 

235 

236 # final calibrations and statistics 

237 astrometry = pexConfig.ConfigurableField( 

238 target=lsst.meas.astrom.AstrometryTask, 

239 doc="Task to perform astrometric calibration to fit a WCS.", 

240 ) 

241 astrometry_ref_loader = pexConfig.ConfigField( 

242 dtype=lsst.meas.algorithms.LoadReferenceObjectsConfig, 

243 doc="Configuration of reference object loader for astrometric fit.", 

244 ) 

245 photometry = pexConfig.ConfigurableField( 

246 target=photoCal.PhotoCalTask, 

247 doc="Task to perform photometric calibration to fit a PhotoCalib.", 

248 ) 

249 photometry_ref_loader = pexConfig.ConfigField( 

250 dtype=lsst.meas.algorithms.LoadReferenceObjectsConfig, 

251 doc="Configuration of reference object loader for photometric fit.", 

252 ) 

253 

254 compute_summary_stats = pexConfig.ConfigurableField( 

255 target=computeExposureSummaryStats.ComputeExposureSummaryStatsTask, 

256 doc="Task to to compute summary statistics on the calibrated exposure." 

257 ) 

258 

259 def setDefaults(self): 

260 super().setDefaults() 

261 

262 # Use a very broad PSF here, to throughly reject CRs. 

263 # TODO investigation: a large initial psf guess may make stars look 

264 # like CRs for very good seeing images. 

265 self.install_simple_psf.fwhm = 4 

266 

267 # S/N>=50 sources for PSF determination, but detection to S/N=5. 

268 self.psf_detection.thresholdValue = 5.0 

269 self.psf_detection.includeThresholdMultiplier = 10.0 

270 # TODO investigation: Probably want False here, but that may require 

271 # tweaking the background spatial scale, to make it small enough to 

272 # prevent extra peaks in the wings of bright objects. 

273 self.psf_detection.doTempLocalBackground = False 

274 # NOTE: we do want reEstimateBackground=True in psf_detection, so that 

275 # each measurement step is done with the best background available. 

276 

277 # Minimal measurement plugins for PSF determination. 

278 # TODO DM-39203: We can drop GaussianFlux and PsfFlux, if we use 

279 # shapeHSM/moments for star/galaxy separation. 

280 # TODO DM-39203: we can remove aperture correction from this task once 

281 # we are using the shape-based star/galaxy code. 

282 self.psf_source_measurement.plugins = ["base_PixelFlags", 

283 "base_SdssCentroid", 

284 "ext_shapeHSM_HsmSourceMoments", 

285 "base_CircularApertureFlux", 

286 "base_GaussianFlux", 

287 "base_PsfFlux", 

288 ] 

289 self.psf_source_measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

290 # Only measure apertures we need for PSF measurement. 

291 self.psf_source_measurement.plugins["base_CircularApertureFlux"].radii = [12.0] 

292 # TODO DM-40843: Remove this line once this is the psfex default. 

293 self.psf_measure_psf.psfDeterminer["psfex"].photometricFluxField = \ 

294 "base_CircularApertureFlux_12_0_instFlux" 

295 

296 # No extendeness information available: we need the aperture 

297 # corrections to determine that. 

298 self.measure_aperture_correction.sourceSelector["science"].doUnresolved = False 

299 self.measure_aperture_correction.sourceSelector["science"].flags.good = ["calib_psf_used"] 

300 self.measure_aperture_correction.sourceSelector["science"].flags.bad = [] 

301 

302 # Detection for good S/N for astrometry/photometry and other 

303 # downstream tasks; detection mask to S/N>=5, but S/N>=10 peaks. 

304 self.star_detection.thresholdValue = 5.0 

305 self.star_detection.includeThresholdMultiplier = 2.0 

306 self.star_measurement.plugins = ["base_PixelFlags", 

307 "base_SdssCentroid", 

308 "ext_shapeHSM_HsmSourceMoments", 

309 'ext_shapeHSM_HsmPsfMoments', 

310 "base_GaussianFlux", 

311 "base_PsfFlux", 

312 "base_CircularApertureFlux", 

313 "base_ClassificationSizeExtendedness", 

314 ] 

315 self.star_measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

316 self.star_measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

317 # Only measure the apertures we need for star selection. 

318 self.star_measurement.plugins["base_CircularApertureFlux"].radii = [12.0] 

319 

320 # Select isolated stars with reliable measurements and no bad flags. 

321 self.star_selector["science"].doFlags = True 

322 self.star_selector["science"].doUnresolved = True 

323 self.star_selector["science"].doSignalToNoise = True 

324 self.star_selector["science"].doIsolated = True 

325 self.star_selector["science"].signalToNoise.minimum = 10.0 

326 # Keep sky sources in the output catalog, even though they aren't 

327 # wanted for calibration. 

328 self.star_selector["science"].doSkySources = True 

329 

330 # Use the affine WCS fitter (assumes we have a good camera geometry). 

331 self.astrometry.wcsFitter.retarget(lsst.meas.astrom.FitAffineWcsTask) 

332 # phot_g_mean is the primary Gaia band for all input bands. 

333 self.astrometry_ref_loader.anyFilterMapsToThis = "phot_g_mean" 

334 

335 # Only reject sky sources; we already selected good stars. 

336 self.astrometry.sourceSelector["science"].doFlags = True 

337 self.astrometry.sourceSelector["science"].flags.bad = ["sky_source"] 

338 self.photometry.match.sourceSelection.doFlags = True 

339 self.photometry.match.sourceSelection.flags.bad = ["sky_source"] 

340 

341 # All sources should be good for PSF summary statistics. 

342 # TODO: These should both be changed to calib_psf_used with DM-41640. 

343 self.compute_summary_stats.starSelection = "calib_photometry_used" 

344 self.compute_summary_stats.starSelector.flags.good = ["calib_photometry_used"] 

345 

346 

347class CalibrateImageTask(pipeBase.PipelineTask): 

348 """Compute the PSF, aperture corrections, astrometric and photometric 

349 calibrations, and summary statistics for a single science exposure, and 

350 produce a catalog of brighter stars that were used to calibrate it. 

351 

352 Parameters 

353 ---------- 

354 initial_stars_schema : `lsst.afw.table.Schema` 

355 Schema of the initial_stars output catalog. 

356 """ 

357 _DefaultName = "calibrateImage" 

358 ConfigClass = CalibrateImageConfig 

359 

360 def __init__(self, initial_stars_schema=None, **kwargs): 

361 super().__init__(**kwargs) 

362 

363 self.makeSubtask("snap_combine") 

364 

365 # PSF determination subtasks 

366 self.makeSubtask("install_simple_psf") 

367 self.makeSubtask("psf_repair") 

368 self.makeSubtask("psf_subtract_background") 

369 self.psf_schema = afwTable.SourceTable.makeMinimalSchema() 

370 self.makeSubtask("psf_detection", schema=self.psf_schema) 

371 self.makeSubtask("psf_source_measurement", schema=self.psf_schema) 

372 self.makeSubtask("psf_measure_psf", schema=self.psf_schema) 

373 

374 self.makeSubtask("measure_aperture_correction", schema=self.psf_schema) 

375 

376 # star measurement subtasks 

377 if initial_stars_schema is None: 

378 initial_stars_schema = afwTable.SourceTable.makeMinimalSchema() 

379 

380 # These fields let us track which sources were used for psf and 

381 # aperture correction calculations. 

382 self.psf_fields = ("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved", 

383 # TODO DM-39203: these can be removed once apcorr is gone. 

384 "apcorr_slot_CalibFlux_used", "apcorr_base_GaussianFlux_used", 

385 "apcorr_base_PsfFlux_used") 

386 for field in self.psf_fields: 

387 item = self.psf_schema.find(field) 

388 initial_stars_schema.addField(item.getField()) 

389 

390 afwTable.CoordKey.addErrorFields(initial_stars_schema) 

391 self.makeSubtask("star_detection", schema=initial_stars_schema) 

392 self.makeSubtask("star_sky_sources", schema=initial_stars_schema) 

393 self.makeSubtask("star_deblend", schema=initial_stars_schema) 

394 self.makeSubtask("star_measurement", schema=initial_stars_schema) 

395 self.makeSubtask("star_apply_aperture_correction", schema=initial_stars_schema) 

396 self.makeSubtask("star_catalog_calculation", schema=initial_stars_schema) 

397 self.makeSubtask("star_set_primary_flags", schema=initial_stars_schema, isSingleFrame=True) 

398 self.makeSubtask("star_selector") 

399 

400 self.makeSubtask("astrometry", schema=initial_stars_schema) 

401 self.makeSubtask("photometry", schema=initial_stars_schema) 

402 

403 self.makeSubtask("compute_summary_stats") 

404 

405 # For the butler to persist it. 

406 self.initial_stars_schema = afwTable.SourceCatalog(initial_stars_schema) 

407 

408 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

409 inputs = butlerQC.get(inputRefs) 

410 exposures = inputs.pop("exposures") 

411 

412 id_generator = self.config.id_generator.apply(butlerQC.quantum.dataId) 

413 

414 astrometry_loader = lsst.meas.algorithms.ReferenceObjectLoader( 

415 dataIds=[ref.datasetRef.dataId for ref in inputRefs.astrometry_ref_cat], 

416 refCats=inputs.pop("astrometry_ref_cat"), 

417 name=self.config.connections.astrometry_ref_cat, 

418 config=self.config.astrometry_ref_loader, log=self.log) 

419 self.astrometry.setRefObjLoader(astrometry_loader) 

420 

421 photometry_loader = lsst.meas.algorithms.ReferenceObjectLoader( 

422 dataIds=[ref.datasetRef.dataId for ref in inputRefs.photometry_ref_cat], 

423 refCats=inputs.pop("photometry_ref_cat"), 

424 name=self.config.connections.photometry_ref_cat, 

425 config=self.config.photometry_ref_loader, log=self.log) 

426 self.photometry.match.setRefObjLoader(photometry_loader) 

427 

428 # This should not happen with a properly configured execution context. 

429 assert not inputs, "runQuantum got more inputs than expected" 

430 

431 # Specify the fields that `annotate` needs below, to ensure they 

432 # exist, even as None. 

433 result = pipeBase.Struct(exposure=None, 

434 stars_footprints=None, 

435 psf_stars_footprints=None, 

436 ) 

437 try: 

438 self.run(exposures=exposures, result=result, id_generator=id_generator) 

439 except pipeBase.AlgorithmError as e: 

440 error = pipeBase.AnnotatedPartialOutputsError.annotate( 

441 e, 

442 self, 

443 result.exposure, 

444 result.psf_stars_footprints, 

445 result.stars_footprints, 

446 log=self.log 

447 ) 

448 butlerQC.put(result, outputRefs) 

449 raise error from e 

450 

451 butlerQC.put(result, outputRefs) 

452 

453 @timeMethod 

454 def run(self, *, exposures, id_generator=None, result=None): 

455 """Find stars and perform psf measurement, then do a deeper detection 

456 and measurement and calibrate astrometry and photometry from that. 

457 

458 Parameters 

459 ---------- 

460 exposures : `lsst.afw.image.Exposure` or `list` [`lsst.afw.image.Exposure`] 

461 Post-ISR exposure(s), with an initial WCS, VisitInfo, and Filter. 

462 Modified in-place during processing if only one is passed. 

463 If two exposures are passed, treat them as snaps and combine 

464 before doing further processing. 

465 id_generator : `lsst.meas.base.IdGenerator`, optional 

466 Object that generates source IDs and provides random seeds. 

467 result : `lsst.pipe.base.Struct`, optional 

468 Result struct that is modified to allow saving of partial outputs 

469 for some failure conditions. If the task completes successfully, 

470 this is also returned. 

471 

472 Returns 

473 ------- 

474 result : `lsst.pipe.base.Struct` 

475 Results as a struct with attributes: 

476 

477 ``exposure`` 

478 Calibrated exposure, with pixels in nJy units. 

479 (`lsst.afw.image.Exposure`) 

480 ``stars`` 

481 Stars that were used to calibrate the exposure, with 

482 calibrated fluxes and magnitudes. 

483 (`astropy.table.Table`) 

484 ``stars_footprints`` 

485 Footprints of stars that were used to calibrate the exposure. 

486 (`lsst.afw.table.SourceCatalog`) 

487 ``psf_stars`` 

488 Stars that were used to determine the image PSF. 

489 (`astropy.table.Table`) 

490 ``psf_stars_footprints`` 

491 Footprints of stars that were used to determine the image PSF. 

492 (`lsst.afw.table.SourceCatalog`) 

493 ``background`` 

494 Background that was fit to the exposure when detecting 

495 ``stars``. (`lsst.afw.math.BackgroundList`) 

496 ``applied_photo_calib`` 

497 Photometric calibration that was fit to the star catalog and 

498 applied to the exposure. (`lsst.afw.image.PhotoCalib`) 

499 ``astrometry_matches`` 

500 Reference catalog stars matches used in the astrometric fit. 

501 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`) 

502 ``photometry_matches`` 

503 Reference catalog stars matches used in the photometric fit. 

504 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`) 

505 """ 

506 if result is None: 

507 result = pipeBase.Struct() 

508 if id_generator is None: 

509 id_generator = lsst.meas.base.IdGenerator() 

510 

511 result.exposure = self._handle_snaps(exposures) 

512 

513 # TODO remove on DM-43083: work around the fact that we don't want 

514 # to run streak detection in this task in production. 

515 result.exposure.mask.addMaskPlane("STREAK") 

516 

517 result.psf_stars_footprints, result.background, candidates = self._compute_psf(result.exposure, 

518 id_generator) 

519 result.psf_stars = result.psf_stars_footprints.asAstropy() 

520 

521 self._measure_aperture_correction(result.exposure, result.psf_stars) 

522 

523 result.stars_footprints = self._find_stars(result.exposure, result.background, id_generator) 

524 self._match_psf_stars(result.psf_stars_footprints, result.stars_footprints) 

525 result.stars = result.stars_footprints.asAstropy() 

526 

527 astrometry_matches, astrometry_meta = self._fit_astrometry(result.exposure, result.stars_footprints) 

528 if self.config.optional_outputs: 

529 result.astrometry_matches = lsst.meas.astrom.denormalizeMatches(astrometry_matches, 

530 astrometry_meta) 

531 

532 result.stars_footprints, photometry_matches, \ 

533 photometry_meta, result.applied_photo_calib = self._fit_photometry(result.exposure, 

534 result.stars_footprints) 

535 # fit_photometry returns a new catalog, so we need a new astropy table view. 

536 result.stars = result.stars_footprints.asAstropy() 

537 if self.config.optional_outputs: 

538 result.photometry_matches = lsst.meas.astrom.denormalizeMatches(photometry_matches, 

539 photometry_meta) 

540 

541 self._summarize(result.exposure, result.stars_footprints, result.background) 

542 

543 return result 

544 

545 def _handle_snaps(self, exposure): 

546 """Combine two snaps into one exposure, or return a single exposure. 

547 

548 Parameters 

549 ---------- 

550 exposure : `lsst.afw.image.Exposure` or `list` [`lsst.afw.image.Exposure]` 

551 One or two exposures to combine as snaps. 

552 

553 Returns 

554 ------- 

555 exposure : `lsst.afw.image.Exposure` 

556 A single exposure to continue processing. 

557 

558 Raises 

559 ------ 

560 RuntimeError 

561 Raised if input does not contain either 1 or 2 exposures. 

562 """ 

563 if isinstance(exposure, lsst.afw.image.Exposure): 

564 return exposure 

565 

566 if isinstance(exposure, collections.abc.Sequence): 

567 match len(exposure): 

568 case 1: 

569 return exposure[0] 

570 case 2: 

571 return self.snap_combine.run(exposure[0], exposure[1]).exposure 

572 case n: 

573 raise RuntimeError(f"Can only process 1 or 2 snaps, not {n}.") 

574 

575 def _compute_psf(self, exposure, id_generator): 

576 """Find bright sources detected on an exposure and fit a PSF model to 

577 them, repairing likely cosmic rays before detection. 

578 

579 Repair, detect, measure, and compute PSF twice, to ensure the PSF 

580 model does not include contributions from cosmic rays. 

581 

582 Parameters 

583 ---------- 

584 exposure : `lsst.afw.image.Exposure` 

585 Exposure to detect and measure bright stars on. 

586 id_generator : `lsst.meas.base.IdGenerator`, optional 

587 Object that generates source IDs and provides random seeds. 

588 

589 Returns 

590 ------- 

591 sources : `lsst.afw.table.SourceCatalog` 

592 Catalog of detected bright sources. 

593 background : `lsst.afw.math.BackgroundList` 

594 Background that was fit to the exposure during detection. 

595 cell_set : `lsst.afw.math.SpatialCellSet` 

596 PSF candidates returned by the psf determiner. 

597 """ 

598 def log_psf(msg): 

599 """Log the parameters of the psf and background, with a prepended 

600 message. 

601 """ 

602 position = exposure.psf.getAveragePosition() 

603 sigma = exposure.psf.computeShape(position).getDeterminantRadius() 

604 dimensions = exposure.psf.computeImage(position).getDimensions() 

605 median_background = np.median(background.getImage().array) 

606 self.log.info("%s sigma=%0.4f, dimensions=%s; median background=%0.2f", 

607 msg, sigma, dimensions, median_background) 

608 

609 self.log.info("First pass detection with Guassian PSF FWHM=%s pixels", 

610 self.config.install_simple_psf.fwhm) 

611 self.install_simple_psf.run(exposure=exposure) 

612 

613 background = self.psf_subtract_background.run(exposure=exposure).background 

614 log_psf("Initial PSF:") 

615 self.psf_repair.run(exposure=exposure, keepCRs=True) 

616 

617 table = afwTable.SourceTable.make(self.psf_schema, id_generator.make_table_id_factory()) 

618 # Re-estimate the background during this detection step, so that 

619 # measurement uses the most accurate background-subtraction. 

620 detections = self.psf_detection.run(table=table, exposure=exposure, background=background) 

621 self.psf_source_measurement.run(detections.sources, exposure) 

622 psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources) 

623 # Replace the initial PSF with something simpler for the second 

624 # repair/detect/measure/measure_psf step: this can help it converge. 

625 self.install_simple_psf.run(exposure=exposure) 

626 

627 log_psf("Rerunning with simple PSF:") 

628 # TODO investigation: Should we only re-run repair here, to use the 

629 # new PSF? Maybe we *do* need to re-run measurement with PsfFlux, to 

630 # use the fitted PSF? 

631 # TODO investigation: do we need a separate measurement task here 

632 # for the post-psf_measure_psf step, since we only want to do PsfFlux 

633 # and GaussianFlux *after* we have a PSF? Maybe that's not relevant 

634 # once DM-39203 is merged? 

635 self.psf_repair.run(exposure=exposure, keepCRs=True) 

636 # Re-estimate the background during this detection step, so that 

637 # measurement uses the most accurate background-subtraction. 

638 detections = self.psf_detection.run(table=table, exposure=exposure, background=background) 

639 self.psf_source_measurement.run(detections.sources, exposure) 

640 psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources) 

641 

642 log_psf("Final PSF:") 

643 

644 # Final repair with final PSF, removing cosmic rays this time. 

645 self.psf_repair.run(exposure=exposure) 

646 # Final measurement with the CRs removed. 

647 self.psf_source_measurement.run(detections.sources, exposure) 

648 

649 # PSF is set on exposure; only return candidates for optional saving. 

650 return detections.sources, background, psf_result.cellSet 

651 

652 def _measure_aperture_correction(self, exposure, bright_sources): 

653 """Measure and set the ApCorrMap on the Exposure, using 

654 previously-measured bright sources. 

655 

656 Parameters 

657 ---------- 

658 exposure : `lsst.afw.image.Exposure` 

659 Exposure to set the ApCorrMap on. 

660 bright_sources : `lsst.afw.table.SourceCatalog` 

661 Catalog of detected bright sources; modified to include columns 

662 necessary for point source determination for the aperture correction 

663 calculation. 

664 """ 

665 result = self.measure_aperture_correction.run(exposure, bright_sources) 

666 exposure.setApCorrMap(result.apCorrMap) 

667 

668 def _find_stars(self, exposure, background, id_generator): 

669 """Detect stars on an exposure that has a PSF model, and measure their 

670 PSF, circular aperture, compensated gaussian fluxes. 

671 

672 Parameters 

673 ---------- 

674 exposure : `lsst.afw.image.Exposure` 

675 Exposure to set the ApCorrMap on. 

676 background : `lsst.afw.math.BackgroundList` 

677 Background that was fit to the exposure during detection; 

678 modified in-place during subsequent detection. 

679 id_generator : `lsst.meas.base.IdGenerator` 

680 Object that generates source IDs and provides random seeds. 

681 

682 Returns 

683 ------- 

684 stars : `SourceCatalog` 

685 Sources that are very likely to be stars, with a limited set of 

686 measurements performed on them. 

687 """ 

688 table = afwTable.SourceTable.make(self.initial_stars_schema.schema, 

689 id_generator.make_table_id_factory()) 

690 # Re-estimate the background during this detection step, so that 

691 # measurement uses the most accurate background-subtraction. 

692 detections = self.star_detection.run(table=table, exposure=exposure, background=background) 

693 sources = detections.sources 

694 self.star_sky_sources.run(exposure.mask, id_generator.catalog_id, sources) 

695 

696 # TODO investigation: Could this deblender throw away blends of non-PSF sources? 

697 self.star_deblend.run(exposure=exposure, sources=sources) 

698 # The deblender may not produce a contiguous catalog; ensure 

699 # contiguity for subsequent tasks. 

700 if not sources.isContiguous(): 

701 sources = sources.copy(deep=True) 

702 

703 # Measure everything, and use those results to select only stars. 

704 self.star_measurement.run(sources, exposure) 

705 self.star_apply_aperture_correction.run(sources, exposure.info.getApCorrMap()) 

706 self.star_catalog_calculation.run(sources) 

707 self.star_set_primary_flags.run(sources) 

708 

709 result = self.star_selector.run(sources) 

710 # The star selector may not produce a contiguous catalog. 

711 if not result.sourceCat.isContiguous(): 

712 return result.sourceCat.copy(deep=True) 

713 else: 

714 return result.sourceCat 

715 

716 def _match_psf_stars(self, psf_stars, stars): 

717 """Match calibration stars to psf stars, to identify which were psf 

718 candidates, and which were used or reserved during psf measurement. 

719 

720 Parameters 

721 ---------- 

722 psf_stars : `lsst.afw.table.SourceCatalog` 

723 PSF candidate stars that were sent to the psf determiner. Used to 

724 populate psf-related flag fields. 

725 stars : `lsst.afw.table.SourceCatalog` 

726 Stars that will be used for calibration; psf-related fields will 

727 be updated in-place. 

728 

729 Notes 

730 ----- 

731 This code was adapted from CalibrateTask.copyIcSourceFields(). 

732 """ 

733 control = afwTable.MatchControl() 

734 # Return all matched objects, to separate blends. 

735 control.findOnlyClosest = False 

736 matches = afwTable.matchXy(psf_stars, stars, 3.0, control) 

737 deblend_key = stars.schema["deblend_nChild"].asKey() 

738 matches = [m for m in matches if m[1].get(deblend_key) == 0] 

739 

740 # Because we had to allow multiple matches to handle parents, we now 

741 # need to prune to the best (closest) matches. 

742 # Closest matches is a dict of psf_stars source ID to Match record 

743 # (psf_stars source, sourceCat source, distance in pixels). 

744 best = {} 

745 for match_psf, match_stars, d in matches: 

746 match = best.get(match_psf.getId()) 

747 if match is None or d <= match[2]: 

748 best[match_psf.getId()] = (match_psf, match_stars, d) 

749 matches = list(best.values()) 

750 # We'll use this to construct index arrays into each catalog. 

751 ids = np.array([(match_psf.getId(), match_stars.getId()) for match_psf, match_stars, d in matches]).T 

752 

753 # Check that no stars sources are listed twice; we already know 

754 # that each match has a unique psf_stars id, due to using as the key 

755 # in best above. 

756 n_matches = len(matches) 

757 n_unique = len(set(m[1].getId() for m in matches)) 

758 if n_unique != n_matches: 

759 self.log.warning("%d psf_stars matched only %d stars; ", 

760 n_matches, n_unique) 

761 if n_matches == 0: 

762 msg = (f"0 psf_stars out of {len(psf_stars)} matched {len(stars)} calib stars." 

763 " Downstream processes probably won't have useful stars in this case." 

764 " Is `star_source_selector` too strict?") 

765 # TODO DM-39842: Turn this into an AlgorithmicError. 

766 raise RuntimeError(msg) 

767 

768 # The indices of the IDs, so we can update the flag fields as arrays. 

769 idx_psf_stars = np.searchsorted(psf_stars["id"], ids[0]) 

770 idx_stars = np.searchsorted(stars["id"], ids[1]) 

771 for field in self.psf_fields: 

772 result = np.zeros(len(stars), dtype=bool) 

773 result[idx_stars] = psf_stars[field][idx_psf_stars] 

774 stars[field] = result 

775 

776 def _fit_astrometry(self, exposure, stars): 

777 """Fit an astrometric model to the data and return the reference 

778 matches used in the fit, and the fitted WCS. 

779 

780 Parameters 

781 ---------- 

782 exposure : `lsst.afw.image.Exposure` 

783 Exposure that is being fit, to get PSF and other metadata from. 

784 Modified to add the fitted skyWcs. 

785 stars : `SourceCatalog` 

786 Good stars selected for use in calibration, with RA/Dec coordinates 

787 computed from the pixel positions and fitted WCS. 

788 

789 Returns 

790 ------- 

791 matches : `list` [`lsst.afw.table.ReferenceMatch`] 

792 Reference/stars matches used in the fit. 

793 """ 

794 result = self.astrometry.run(stars, exposure) 

795 return result.matches, result.matchMeta 

796 

797 def _fit_photometry(self, exposure, stars): 

798 """Fit a photometric model to the data and return the reference 

799 matches used in the fit, and the fitted PhotoCalib. 

800 

801 Parameters 

802 ---------- 

803 exposure : `lsst.afw.image.Exposure` 

804 Exposure that is being fit, to get PSF and other metadata from. 

805 Modified to be in nanojanksy units, with an assigned photoCalib 

806 identically 1. 

807 stars : `lsst.afw.table.SourceCatalog` 

808 Good stars selected for use in calibration. 

809 

810 Returns 

811 ------- 

812 calibrated_stars : `lsst.afw.table.SourceCatalog` 

813 Star catalog with flux/magnitude columns computed from the fitted 

814 photoCalib. 

815 matches : `list` [`lsst.afw.table.ReferenceMatch`] 

816 Reference/stars matches used in the fit. 

817 photoCalib : `lsst.afw.image.PhotoCalib` 

818 Photometric calibration that was fit to the star catalog. 

819 """ 

820 result = self.photometry.run(exposure, stars) 

821 calibrated_stars = result.photoCalib.calibrateCatalog(stars) 

822 exposure.maskedImage = result.photoCalib.calibrateImage(exposure.maskedImage) 

823 identity = afwImage.PhotoCalib(1.0, 

824 result.photoCalib.getCalibrationErr(), 

825 bbox=exposure.getBBox()) 

826 exposure.setPhotoCalib(identity) 

827 

828 return calibrated_stars, result.matches, result.matchMeta, result.photoCalib 

829 

830 def _summarize(self, exposure, stars, background): 

831 """Compute summary statistics on the exposure and update in-place the 

832 calibrations attached to it. 

833 

834 Parameters 

835 ---------- 

836 exposure : `lsst.afw.image.Exposure` 

837 Exposure that was calibrated, to get PSF and other metadata from. 

838 Modified to contain the computed summary statistics. 

839 stars : `SourceCatalog` 

840 Good stars selected used in calibration. 

841 background : `lsst.afw.math.BackgroundList` 

842 Background that was fit to the exposure during detection of the 

843 above stars. 

844 """ 

845 # TODO investigation: because this takes the photoCalib from the 

846 # exposure, photometric summary values may be "incorrect" (i.e. they 

847 # will reflect the ==1 nJy calibration on the exposure, not the 

848 # applied calibration). This needs to be checked. 

849 summary = self.compute_summary_stats.run(exposure, stars, background) 

850 exposure.info.setSummaryStats(summary)