Coverage for python/lsst/pipe/tasks/calibrateImage.py: 24%

263 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-14 09:29 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections.abc 

23 

24import numpy as np 

25 

26import lsst.afw.table as afwTable 

27import lsst.afw.image as afwImage 

28import lsst.meas.algorithms 

29import lsst.meas.algorithms.installGaussianPsf 

30import lsst.meas.algorithms.measureApCorr 

31import lsst.meas.algorithms.setPrimaryFlags 

32import lsst.meas.base 

33import lsst.meas.astrom 

34import lsst.meas.deblender 

35import lsst.meas.extensions.shapeHSM 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38from lsst.pipe.base import connectionTypes 

39from lsst.utils.timer import timeMethod 

40 

41from . import measurePsf, repair, photoCal, computeExposureSummaryStats, snapCombine 

42 

43 

44class CalibrateImageConnections(pipeBase.PipelineTaskConnections, 

45 dimensions=("instrument", "visit", "detector")): 

46 

47 astrometry_ref_cat = connectionTypes.PrerequisiteInput( 

48 doc="Reference catalog to use for astrometric calibration.", 

49 name="gaia_dr3_20230707", 

50 storageClass="SimpleCatalog", 

51 dimensions=("skypix",), 

52 deferLoad=True, 

53 multiple=True, 

54 ) 

55 photometry_ref_cat = connectionTypes.PrerequisiteInput( 

56 doc="Reference catalog to use for photometric calibration.", 

57 name="ps1_pv3_3pi_20170110", 

58 storageClass="SimpleCatalog", 

59 dimensions=("skypix",), 

60 deferLoad=True, 

61 multiple=True 

62 ) 

63 

64 exposures = connectionTypes.Input( 

65 doc="Exposure (or two snaps) to be calibrated, and detected and measured on.", 

66 name="postISRCCD", 

67 storageClass="Exposure", 

68 multiple=True, # to handle 1 exposure or 2 snaps 

69 dimensions=["instrument", "exposure", "detector"], 

70 ) 

71 

72 # outputs 

73 initial_stars_schema = connectionTypes.InitOutput( 

74 doc="Schema of the output initial stars catalog.", 

75 name="initial_stars_schema", 

76 storageClass="SourceCatalog", 

77 ) 

78 

79 # TODO DM-38732: We want some kind of flag on Exposures/Catalogs to make 

80 # it obvious which components had failed to be computed/persisted. 

81 exposure = connectionTypes.Output( 

82 doc="Photometrically calibrated exposure with fitted calibrations and summary statistics.", 

83 name="initial_pvi", 

84 storageClass="ExposureF", 

85 dimensions=("instrument", "visit", "detector"), 

86 ) 

87 stars = connectionTypes.Output( 

88 doc="Catalog of unresolved sources detected on the calibrated exposure.", 

89 name="initial_stars_detector", 

90 storageClass="ArrowAstropy", 

91 dimensions=["instrument", "visit", "detector"], 

92 ) 

93 stars_footprints = connectionTypes.Output( 

94 doc="Catalog of unresolved sources detected on the calibrated exposure; " 

95 "includes source footprints.", 

96 name="initial_stars_footprints_detector", 

97 storageClass="SourceCatalog", 

98 dimensions=["instrument", "visit", "detector"], 

99 ) 

100 applied_photo_calib = connectionTypes.Output( 

101 doc="Photometric calibration that was applied to exposure.", 

102 name="initial_photoCalib_detector", 

103 storageClass="PhotoCalib", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 background = connectionTypes.Output( 

107 doc="Background models estimated during calibration task.", 

108 name="initial_pvi_background", 

109 storageClass="Background", 

110 dimensions=("instrument", "visit", "detector"), 

111 ) 

112 

113 # Optional outputs 

114 psf_stars_footprints = connectionTypes.Output( 

115 doc="Catalog of bright unresolved sources detected on the exposure used for PSF determination; " 

116 "includes source footprints.", 

117 name="initial_psf_stars_footprints_detector", 

118 storageClass="SourceCatalog", 

119 dimensions=["instrument", "visit", "detector"], 

120 ) 

121 psf_stars = connectionTypes.Output( 

122 doc="Catalog of bright unresolved sources detected on the exposure used for PSF determination.", 

123 name="initial_psf_stars_detector", 

124 storageClass="ArrowAstropy", 

125 dimensions=["instrument", "visit", "detector"], 

126 ) 

127 astrometry_matches = connectionTypes.Output( 

128 doc="Source to reference catalog matches from the astrometry solver.", 

129 name="initial_astrometry_match_detector", 

130 storageClass="Catalog", 

131 dimensions=("instrument", "visit", "detector"), 

132 ) 

133 photometry_matches = connectionTypes.Output( 

134 doc="Source to reference catalog matches from the photometry solver.", 

135 name="initial_photometry_match_detector", 

136 storageClass="Catalog", 

137 dimensions=("instrument", "visit", "detector"), 

138 ) 

139 

140 def __init__(self, *, config=None): 

141 super().__init__(config=config) 

142 if not config.optional_outputs: 

143 del self.psf_stars 

144 del self.psf_stars_footprints 

145 del self.astrometry_matches 

146 del self.photometry_matches 

147 

148 

149class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateImageConnections): 

150 optional_outputs = pexConfig.ListField( 

151 doc="Which optional outputs to save (as their connection name)?", 

152 dtype=str, 

153 # TODO: note somewhere to disable this for benchmarking, but should 

154 # we always have it on for production runs? 

155 default=["psf_stars", "psf_stars_footprints", "astrometry_matches", "photometry_matches"], 

156 optional=True 

157 ) 

158 

159 # To generate catalog ids consistently across subtasks. 

160 id_generator = lsst.meas.base.DetectorVisitIdGeneratorConfig.make_field() 

161 

162 snap_combine = pexConfig.ConfigurableField( 

163 target=snapCombine.SnapCombineTask, 

164 doc="Task to combine two snaps to make one exposure.", 

165 ) 

166 

167 # subtasks used during psf characterization 

168 install_simple_psf = pexConfig.ConfigurableField( 

169 target=lsst.meas.algorithms.installGaussianPsf.InstallGaussianPsfTask, 

170 doc="Task to install a simple PSF model into the input exposure to use " 

171 "when detecting bright sources for PSF estimation.", 

172 ) 

173 psf_repair = pexConfig.ConfigurableField( 

174 target=repair.RepairTask, 

175 doc="Task to repair cosmic rays on the exposure before PSF determination.", 

176 ) 

177 psf_subtract_background = pexConfig.ConfigurableField( 

178 target=lsst.meas.algorithms.SubtractBackgroundTask, 

179 doc="Task to perform intial background subtraction, before first detection pass.", 

180 ) 

181 psf_detection = pexConfig.ConfigurableField( 

182 target=lsst.meas.algorithms.SourceDetectionTask, 

183 doc="Task to detect sources for PSF determination." 

184 ) 

185 psf_source_measurement = pexConfig.ConfigurableField( 

186 target=lsst.meas.base.SingleFrameMeasurementTask, 

187 doc="Task to measure sources to be used for psf estimation." 

188 ) 

189 psf_measure_psf = pexConfig.ConfigurableField( 

190 target=measurePsf.MeasurePsfTask, 

191 doc="Task to measure the psf on bright sources." 

192 ) 

193 

194 # TODO DM-39203: we can remove aperture correction from this task once we are 

195 # using the shape-based star/galaxy code. 

196 measure_aperture_correction = pexConfig.ConfigurableField( 

197 target=lsst.meas.algorithms.measureApCorr.MeasureApCorrTask, 

198 doc="Task to compute the aperture correction from the bright stars." 

199 ) 

200 

201 # subtasks used during star measurement 

202 star_detection = pexConfig.ConfigurableField( 

203 target=lsst.meas.algorithms.SourceDetectionTask, 

204 doc="Task to detect stars to return in the output catalog." 

205 ) 

206 star_sky_sources = pexConfig.ConfigurableField( 

207 target=lsst.meas.algorithms.SkyObjectsTask, 

208 doc="Task to generate sky sources ('empty' regions where there are no detections).", 

209 ) 

210 star_deblend = pexConfig.ConfigurableField( 

211 target=lsst.meas.deblender.SourceDeblendTask, 

212 doc="Split blended sources into their components." 

213 ) 

214 star_measurement = pexConfig.ConfigurableField( 

215 target=lsst.meas.base.SingleFrameMeasurementTask, 

216 doc="Task to measure stars to return in the output catalog." 

217 ) 

218 star_apply_aperture_correction = pexConfig.ConfigurableField( 

219 target=lsst.meas.base.ApplyApCorrTask, 

220 doc="Task to apply aperture corrections to the selected stars." 

221 ) 

222 star_catalog_calculation = pexConfig.ConfigurableField( 

223 target=lsst.meas.base.CatalogCalculationTask, 

224 doc="Task to compute extendedness values on the star catalog, " 

225 "for the star selector to remove extended sources." 

226 ) 

227 star_set_primary_flags = pexConfig.ConfigurableField( 

228 target=lsst.meas.algorithms.setPrimaryFlags.SetPrimaryFlagsTask, 

229 doc="Task to add isPrimary to the catalog." 

230 ) 

231 star_selector = lsst.meas.algorithms.sourceSelectorRegistry.makeField( 

232 default="science", 

233 doc="Task to select isolated stars to use for calibration." 

234 ) 

235 

236 # final calibrations and statistics 

237 astrometry = pexConfig.ConfigurableField( 

238 target=lsst.meas.astrom.AstrometryTask, 

239 doc="Task to perform astrometric calibration to fit a WCS.", 

240 ) 

241 astrometry_ref_loader = pexConfig.ConfigField( 

242 dtype=lsst.meas.algorithms.LoadReferenceObjectsConfig, 

243 doc="Configuration of reference object loader for astrometric fit.", 

244 ) 

245 photometry = pexConfig.ConfigurableField( 

246 target=photoCal.PhotoCalTask, 

247 doc="Task to perform photometric calibration to fit a PhotoCalib.", 

248 ) 

249 photometry_ref_loader = pexConfig.ConfigField( 

250 dtype=lsst.meas.algorithms.LoadReferenceObjectsConfig, 

251 doc="Configuration of reference object loader for photometric fit.", 

252 ) 

253 

254 compute_summary_stats = pexConfig.ConfigurableField( 

255 target=computeExposureSummaryStats.ComputeExposureSummaryStatsTask, 

256 doc="Task to to compute summary statistics on the calibrated exposure." 

257 ) 

258 

259 def setDefaults(self): 

260 super().setDefaults() 

261 

262 # Use a very broad PSF here, to throughly reject CRs. 

263 # TODO investigation: a large initial psf guess may make stars look 

264 # like CRs for very good seeing images. 

265 self.install_simple_psf.fwhm = 4 

266 

267 # S/N>=50 sources for PSF determination, but detection to S/N=5. 

268 # The thresholdValue sets the minimum flux in a pixel to be included in the 

269 # footprint, while peaks are only detected when they are above 

270 # thresholdValue * includeThresholdMultiplier. The low thresholdValue 

271 # ensures that the footprints are large enough for the noise replacer 

272 # to mask out faint undetected neighbors that are not to be measured. 

273 self.psf_detection.thresholdValue = 5.0 

274 self.psf_detection.includeThresholdMultiplier = 10.0 

275 # TODO investigation: Probably want False here, but that may require 

276 # tweaking the background spatial scale, to make it small enough to 

277 # prevent extra peaks in the wings of bright objects. 

278 self.psf_detection.doTempLocalBackground = False 

279 # NOTE: we do want reEstimateBackground=True in psf_detection, so that 

280 # each measurement step is done with the best background available. 

281 

282 # Minimal measurement plugins for PSF determination. 

283 # TODO DM-39203: We can drop GaussianFlux and PsfFlux, if we use 

284 # shapeHSM/moments for star/galaxy separation. 

285 # TODO DM-39203: we can remove aperture correction from this task once 

286 # we are using the shape-based star/galaxy code. 

287 self.psf_source_measurement.plugins = ["base_PixelFlags", 

288 "base_SdssCentroid", 

289 "ext_shapeHSM_HsmSourceMoments", 

290 "base_CircularApertureFlux", 

291 "base_GaussianFlux", 

292 "base_PsfFlux", 

293 ] 

294 self.psf_source_measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

295 # Only measure apertures we need for PSF measurement. 

296 self.psf_source_measurement.plugins["base_CircularApertureFlux"].radii = [12.0] 

297 # TODO DM-40843: Remove this line once this is the psfex default. 

298 self.psf_measure_psf.psfDeterminer["psfex"].photometricFluxField = \ 

299 "base_CircularApertureFlux_12_0_instFlux" 

300 

301 # No extendeness information available: we need the aperture 

302 # corrections to determine that. 

303 self.measure_aperture_correction.sourceSelector["science"].doUnresolved = False 

304 self.measure_aperture_correction.sourceSelector["science"].flags.good = ["calib_psf_used"] 

305 self.measure_aperture_correction.sourceSelector["science"].flags.bad = [] 

306 

307 # Detection for good S/N for astrometry/photometry and other 

308 # downstream tasks; detection mask to S/N>=5, but S/N>=10 peaks. 

309 self.star_detection.thresholdValue = 5.0 

310 self.star_detection.includeThresholdMultiplier = 2.0 

311 self.star_measurement.plugins = ["base_PixelFlags", 

312 "base_SdssCentroid", 

313 "ext_shapeHSM_HsmSourceMoments", 

314 'ext_shapeHSM_HsmPsfMoments', 

315 "base_GaussianFlux", 

316 "base_PsfFlux", 

317 "base_CircularApertureFlux", 

318 "base_ClassificationSizeExtendedness", 

319 ] 

320 self.star_measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments" 

321 self.star_measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

322 # Only measure the apertures we need for star selection. 

323 self.star_measurement.plugins["base_CircularApertureFlux"].radii = [12.0] 

324 

325 # Select isolated stars with reliable measurements and no bad flags. 

326 self.star_selector["science"].doFlags = True 

327 self.star_selector["science"].doUnresolved = True 

328 self.star_selector["science"].doSignalToNoise = True 

329 self.star_selector["science"].doIsolated = True 

330 self.star_selector["science"].signalToNoise.minimum = 10.0 

331 # Keep sky sources in the output catalog, even though they aren't 

332 # wanted for calibration. 

333 self.star_selector["science"].doSkySources = True 

334 

335 # Use the affine WCS fitter (assumes we have a good camera geometry). 

336 self.astrometry.wcsFitter.retarget(lsst.meas.astrom.FitAffineWcsTask) 

337 # phot_g_mean is the primary Gaia band for all input bands. 

338 self.astrometry_ref_loader.anyFilterMapsToThis = "phot_g_mean" 

339 

340 # Only reject sky sources; we already selected good stars. 

341 self.astrometry.sourceSelector["science"].doFlags = True 

342 self.astrometry.sourceSelector["science"].flags.bad = ["sky_source"] 

343 self.photometry.match.sourceSelection.doFlags = True 

344 self.photometry.match.sourceSelection.flags.bad = ["sky_source"] 

345 

346 # All sources should be good for PSF summary statistics. 

347 # TODO: These should both be changed to calib_psf_used with DM-41640. 

348 self.compute_summary_stats.starSelection = "calib_photometry_used" 

349 self.compute_summary_stats.starSelector.flags.good = ["calib_photometry_used"] 

350 

351 

352class CalibrateImageTask(pipeBase.PipelineTask): 

353 """Compute the PSF, aperture corrections, astrometric and photometric 

354 calibrations, and summary statistics for a single science exposure, and 

355 produce a catalog of brighter stars that were used to calibrate it. 

356 

357 Parameters 

358 ---------- 

359 initial_stars_schema : `lsst.afw.table.Schema` 

360 Schema of the initial_stars output catalog. 

361 """ 

362 _DefaultName = "calibrateImage" 

363 ConfigClass = CalibrateImageConfig 

364 

365 def __init__(self, initial_stars_schema=None, **kwargs): 

366 super().__init__(**kwargs) 

367 

368 self.makeSubtask("snap_combine") 

369 

370 # PSF determination subtasks 

371 self.makeSubtask("install_simple_psf") 

372 self.makeSubtask("psf_repair") 

373 self.makeSubtask("psf_subtract_background") 

374 self.psf_schema = afwTable.SourceTable.makeMinimalSchema() 

375 self.makeSubtask("psf_detection", schema=self.psf_schema) 

376 self.makeSubtask("psf_source_measurement", schema=self.psf_schema) 

377 self.makeSubtask("psf_measure_psf", schema=self.psf_schema) 

378 

379 self.makeSubtask("measure_aperture_correction", schema=self.psf_schema) 

380 

381 # star measurement subtasks 

382 if initial_stars_schema is None: 

383 initial_stars_schema = afwTable.SourceTable.makeMinimalSchema() 

384 

385 # These fields let us track which sources were used for psf and 

386 # aperture correction calculations. 

387 self.psf_fields = ("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved", 

388 # TODO DM-39203: these can be removed once apcorr is gone. 

389 "apcorr_slot_CalibFlux_used", "apcorr_base_GaussianFlux_used", 

390 "apcorr_base_PsfFlux_used") 

391 for field in self.psf_fields: 

392 item = self.psf_schema.find(field) 

393 initial_stars_schema.addField(item.getField()) 

394 

395 afwTable.CoordKey.addErrorFields(initial_stars_schema) 

396 self.makeSubtask("star_detection", schema=initial_stars_schema) 

397 self.makeSubtask("star_sky_sources", schema=initial_stars_schema) 

398 self.makeSubtask("star_deblend", schema=initial_stars_schema) 

399 self.makeSubtask("star_measurement", schema=initial_stars_schema) 

400 self.makeSubtask("star_apply_aperture_correction", schema=initial_stars_schema) 

401 self.makeSubtask("star_catalog_calculation", schema=initial_stars_schema) 

402 self.makeSubtask("star_set_primary_flags", schema=initial_stars_schema, isSingleFrame=True) 

403 self.makeSubtask("star_selector") 

404 

405 self.makeSubtask("astrometry", schema=initial_stars_schema) 

406 self.makeSubtask("photometry", schema=initial_stars_schema) 

407 

408 self.makeSubtask("compute_summary_stats") 

409 

410 # For the butler to persist it. 

411 self.initial_stars_schema = afwTable.SourceCatalog(initial_stars_schema) 

412 

413 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

414 inputs = butlerQC.get(inputRefs) 

415 exposures = inputs.pop("exposures") 

416 

417 id_generator = self.config.id_generator.apply(butlerQC.quantum.dataId) 

418 

419 astrometry_loader = lsst.meas.algorithms.ReferenceObjectLoader( 

420 dataIds=[ref.datasetRef.dataId for ref in inputRefs.astrometry_ref_cat], 

421 refCats=inputs.pop("astrometry_ref_cat"), 

422 name=self.config.connections.astrometry_ref_cat, 

423 config=self.config.astrometry_ref_loader, log=self.log) 

424 self.astrometry.setRefObjLoader(astrometry_loader) 

425 

426 photometry_loader = lsst.meas.algorithms.ReferenceObjectLoader( 

427 dataIds=[ref.datasetRef.dataId for ref in inputRefs.photometry_ref_cat], 

428 refCats=inputs.pop("photometry_ref_cat"), 

429 name=self.config.connections.photometry_ref_cat, 

430 config=self.config.photometry_ref_loader, log=self.log) 

431 self.photometry.match.setRefObjLoader(photometry_loader) 

432 

433 # This should not happen with a properly configured execution context. 

434 assert not inputs, "runQuantum got more inputs than expected" 

435 

436 # Specify the fields that `annotate` needs below, to ensure they 

437 # exist, even as None. 

438 result = pipeBase.Struct(exposure=None, 

439 stars_footprints=None, 

440 psf_stars_footprints=None, 

441 ) 

442 try: 

443 self.run(exposures=exposures, result=result, id_generator=id_generator) 

444 except pipeBase.AlgorithmError as e: 

445 error = pipeBase.AnnotatedPartialOutputsError.annotate( 

446 e, 

447 self, 

448 result.exposure, 

449 result.psf_stars_footprints, 

450 result.stars_footprints, 

451 log=self.log 

452 ) 

453 butlerQC.put(result, outputRefs) 

454 raise error from e 

455 

456 butlerQC.put(result, outputRefs) 

457 

458 @timeMethod 

459 def run(self, *, exposures, id_generator=None, result=None): 

460 """Find stars and perform psf measurement, then do a deeper detection 

461 and measurement and calibrate astrometry and photometry from that. 

462 

463 Parameters 

464 ---------- 

465 exposures : `lsst.afw.image.Exposure` or `list` [`lsst.afw.image.Exposure`] 

466 Post-ISR exposure(s), with an initial WCS, VisitInfo, and Filter. 

467 Modified in-place during processing if only one is passed. 

468 If two exposures are passed, treat them as snaps and combine 

469 before doing further processing. 

470 id_generator : `lsst.meas.base.IdGenerator`, optional 

471 Object that generates source IDs and provides random seeds. 

472 result : `lsst.pipe.base.Struct`, optional 

473 Result struct that is modified to allow saving of partial outputs 

474 for some failure conditions. If the task completes successfully, 

475 this is also returned. 

476 

477 Returns 

478 ------- 

479 result : `lsst.pipe.base.Struct` 

480 Results as a struct with attributes: 

481 

482 ``exposure`` 

483 Calibrated exposure, with pixels in nJy units. 

484 (`lsst.afw.image.Exposure`) 

485 ``stars`` 

486 Stars that were used to calibrate the exposure, with 

487 calibrated fluxes and magnitudes. 

488 (`astropy.table.Table`) 

489 ``stars_footprints`` 

490 Footprints of stars that were used to calibrate the exposure. 

491 (`lsst.afw.table.SourceCatalog`) 

492 ``psf_stars`` 

493 Stars that were used to determine the image PSF. 

494 (`astropy.table.Table`) 

495 ``psf_stars_footprints`` 

496 Footprints of stars that were used to determine the image PSF. 

497 (`lsst.afw.table.SourceCatalog`) 

498 ``background`` 

499 Background that was fit to the exposure when detecting 

500 ``stars``. (`lsst.afw.math.BackgroundList`) 

501 ``applied_photo_calib`` 

502 Photometric calibration that was fit to the star catalog and 

503 applied to the exposure. (`lsst.afw.image.PhotoCalib`) 

504 ``astrometry_matches`` 

505 Reference catalog stars matches used in the astrometric fit. 

506 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`) 

507 ``photometry_matches`` 

508 Reference catalog stars matches used in the photometric fit. 

509 (`list` [`lsst.afw.table.ReferenceMatch`] or `lsst.afw.table.BaseCatalog`) 

510 """ 

511 if result is None: 

512 result = pipeBase.Struct() 

513 if id_generator is None: 

514 id_generator = lsst.meas.base.IdGenerator() 

515 

516 result.exposure = self._handle_snaps(exposures) 

517 

518 # TODO remove on DM-43083: work around the fact that we don't want 

519 # to run streak detection in this task in production. 

520 result.exposure.mask.addMaskPlane("STREAK") 

521 

522 result.psf_stars_footprints, result.background, candidates = self._compute_psf(result.exposure, 

523 id_generator) 

524 result.psf_stars = result.psf_stars_footprints.asAstropy() 

525 

526 self._measure_aperture_correction(result.exposure, result.psf_stars) 

527 

528 result.stars_footprints = self._find_stars(result.exposure, result.background, id_generator) 

529 self._match_psf_stars(result.psf_stars_footprints, result.stars_footprints) 

530 result.stars = result.stars_footprints.asAstropy() 

531 

532 astrometry_matches, astrometry_meta = self._fit_astrometry(result.exposure, result.stars_footprints) 

533 if self.config.optional_outputs: 

534 result.astrometry_matches = lsst.meas.astrom.denormalizeMatches(astrometry_matches, 

535 astrometry_meta) 

536 

537 result.stars_footprints, photometry_matches, \ 

538 photometry_meta, result.applied_photo_calib = self._fit_photometry(result.exposure, 

539 result.stars_footprints) 

540 # fit_photometry returns a new catalog, so we need a new astropy table view. 

541 result.stars = result.stars_footprints.asAstropy() 

542 if self.config.optional_outputs: 

543 result.photometry_matches = lsst.meas.astrom.denormalizeMatches(photometry_matches, 

544 photometry_meta) 

545 

546 self._summarize(result.exposure, result.stars_footprints, result.background) 

547 

548 return result 

549 

550 def _handle_snaps(self, exposure): 

551 """Combine two snaps into one exposure, or return a single exposure. 

552 

553 Parameters 

554 ---------- 

555 exposure : `lsst.afw.image.Exposure` or `list` [`lsst.afw.image.Exposure]` 

556 One or two exposures to combine as snaps. 

557 

558 Returns 

559 ------- 

560 exposure : `lsst.afw.image.Exposure` 

561 A single exposure to continue processing. 

562 

563 Raises 

564 ------ 

565 RuntimeError 

566 Raised if input does not contain either 1 or 2 exposures. 

567 """ 

568 if isinstance(exposure, lsst.afw.image.Exposure): 

569 return exposure 

570 

571 if isinstance(exposure, collections.abc.Sequence): 

572 match len(exposure): 

573 case 1: 

574 return exposure[0] 

575 case 2: 

576 return self.snap_combine.run(exposure[0], exposure[1]).exposure 

577 case n: 

578 raise RuntimeError(f"Can only process 1 or 2 snaps, not {n}.") 

579 

580 def _compute_psf(self, exposure, id_generator): 

581 """Find bright sources detected on an exposure and fit a PSF model to 

582 them, repairing likely cosmic rays before detection. 

583 

584 Repair, detect, measure, and compute PSF twice, to ensure the PSF 

585 model does not include contributions from cosmic rays. 

586 

587 Parameters 

588 ---------- 

589 exposure : `lsst.afw.image.Exposure` 

590 Exposure to detect and measure bright stars on. 

591 id_generator : `lsst.meas.base.IdGenerator`, optional 

592 Object that generates source IDs and provides random seeds. 

593 

594 Returns 

595 ------- 

596 sources : `lsst.afw.table.SourceCatalog` 

597 Catalog of detected bright sources. 

598 background : `lsst.afw.math.BackgroundList` 

599 Background that was fit to the exposure during detection. 

600 cell_set : `lsst.afw.math.SpatialCellSet` 

601 PSF candidates returned by the psf determiner. 

602 """ 

603 def log_psf(msg): 

604 """Log the parameters of the psf and background, with a prepended 

605 message. 

606 """ 

607 position = exposure.psf.getAveragePosition() 

608 sigma = exposure.psf.computeShape(position).getDeterminantRadius() 

609 dimensions = exposure.psf.computeImage(position).getDimensions() 

610 median_background = np.median(background.getImage().array) 

611 self.log.info("%s sigma=%0.4f, dimensions=%s; median background=%0.2f", 

612 msg, sigma, dimensions, median_background) 

613 

614 self.log.info("First pass detection with Guassian PSF FWHM=%s pixels", 

615 self.config.install_simple_psf.fwhm) 

616 self.install_simple_psf.run(exposure=exposure) 

617 

618 background = self.psf_subtract_background.run(exposure=exposure).background 

619 log_psf("Initial PSF:") 

620 self.psf_repair.run(exposure=exposure, keepCRs=True) 

621 

622 table = afwTable.SourceTable.make(self.psf_schema, id_generator.make_table_id_factory()) 

623 # Re-estimate the background during this detection step, so that 

624 # measurement uses the most accurate background-subtraction. 

625 detections = self.psf_detection.run(table=table, exposure=exposure, background=background) 

626 self.psf_source_measurement.run(detections.sources, exposure) 

627 psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources) 

628 # Replace the initial PSF with something simpler for the second 

629 # repair/detect/measure/measure_psf step: this can help it converge. 

630 self.install_simple_psf.run(exposure=exposure) 

631 

632 log_psf("Rerunning with simple PSF:") 

633 # TODO investigation: Should we only re-run repair here, to use the 

634 # new PSF? Maybe we *do* need to re-run measurement with PsfFlux, to 

635 # use the fitted PSF? 

636 # TODO investigation: do we need a separate measurement task here 

637 # for the post-psf_measure_psf step, since we only want to do PsfFlux 

638 # and GaussianFlux *after* we have a PSF? Maybe that's not relevant 

639 # once DM-39203 is merged? 

640 self.psf_repair.run(exposure=exposure, keepCRs=True) 

641 # Re-estimate the background during this detection step, so that 

642 # measurement uses the most accurate background-subtraction. 

643 detections = self.psf_detection.run(table=table, exposure=exposure, background=background) 

644 self.psf_source_measurement.run(detections.sources, exposure) 

645 psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources) 

646 

647 log_psf("Final PSF:") 

648 

649 # Final repair with final PSF, removing cosmic rays this time. 

650 self.psf_repair.run(exposure=exposure) 

651 # Final measurement with the CRs removed. 

652 self.psf_source_measurement.run(detections.sources, exposure) 

653 

654 # PSF is set on exposure; only return candidates for optional saving. 

655 return detections.sources, background, psf_result.cellSet 

656 

657 def _measure_aperture_correction(self, exposure, bright_sources): 

658 """Measure and set the ApCorrMap on the Exposure, using 

659 previously-measured bright sources. 

660 

661 Parameters 

662 ---------- 

663 exposure : `lsst.afw.image.Exposure` 

664 Exposure to set the ApCorrMap on. 

665 bright_sources : `lsst.afw.table.SourceCatalog` 

666 Catalog of detected bright sources; modified to include columns 

667 necessary for point source determination for the aperture correction 

668 calculation. 

669 """ 

670 result = self.measure_aperture_correction.run(exposure, bright_sources) 

671 exposure.setApCorrMap(result.apCorrMap) 

672 

673 def _find_stars(self, exposure, background, id_generator): 

674 """Detect stars on an exposure that has a PSF model, and measure their 

675 PSF, circular aperture, compensated gaussian fluxes. 

676 

677 Parameters 

678 ---------- 

679 exposure : `lsst.afw.image.Exposure` 

680 Exposure to set the ApCorrMap on. 

681 background : `lsst.afw.math.BackgroundList` 

682 Background that was fit to the exposure during detection; 

683 modified in-place during subsequent detection. 

684 id_generator : `lsst.meas.base.IdGenerator` 

685 Object that generates source IDs and provides random seeds. 

686 

687 Returns 

688 ------- 

689 stars : `SourceCatalog` 

690 Sources that are very likely to be stars, with a limited set of 

691 measurements performed on them. 

692 """ 

693 table = afwTable.SourceTable.make(self.initial_stars_schema.schema, 

694 id_generator.make_table_id_factory()) 

695 # Re-estimate the background during this detection step, so that 

696 # measurement uses the most accurate background-subtraction. 

697 detections = self.star_detection.run(table=table, exposure=exposure, background=background) 

698 sources = detections.sources 

699 self.star_sky_sources.run(exposure.mask, id_generator.catalog_id, sources) 

700 

701 # TODO investigation: Could this deblender throw away blends of non-PSF sources? 

702 self.star_deblend.run(exposure=exposure, sources=sources) 

703 # The deblender may not produce a contiguous catalog; ensure 

704 # contiguity for subsequent tasks. 

705 if not sources.isContiguous(): 

706 sources = sources.copy(deep=True) 

707 

708 # Measure everything, and use those results to select only stars. 

709 self.star_measurement.run(sources, exposure) 

710 self.star_apply_aperture_correction.run(sources, exposure.info.getApCorrMap()) 

711 self.star_catalog_calculation.run(sources) 

712 self.star_set_primary_flags.run(sources) 

713 

714 result = self.star_selector.run(sources) 

715 # The star selector may not produce a contiguous catalog. 

716 if not result.sourceCat.isContiguous(): 

717 return result.sourceCat.copy(deep=True) 

718 else: 

719 return result.sourceCat 

720 

721 def _match_psf_stars(self, psf_stars, stars): 

722 """Match calibration stars to psf stars, to identify which were psf 

723 candidates, and which were used or reserved during psf measurement. 

724 

725 Parameters 

726 ---------- 

727 psf_stars : `lsst.afw.table.SourceCatalog` 

728 PSF candidate stars that were sent to the psf determiner. Used to 

729 populate psf-related flag fields. 

730 stars : `lsst.afw.table.SourceCatalog` 

731 Stars that will be used for calibration; psf-related fields will 

732 be updated in-place. 

733 

734 Notes 

735 ----- 

736 This code was adapted from CalibrateTask.copyIcSourceFields(). 

737 """ 

738 control = afwTable.MatchControl() 

739 # Return all matched objects, to separate blends. 

740 control.findOnlyClosest = False 

741 matches = afwTable.matchXy(psf_stars, stars, 3.0, control) 

742 deblend_key = stars.schema["deblend_nChild"].asKey() 

743 matches = [m for m in matches if m[1].get(deblend_key) == 0] 

744 

745 # Because we had to allow multiple matches to handle parents, we now 

746 # need to prune to the best (closest) matches. 

747 # Closest matches is a dict of psf_stars source ID to Match record 

748 # (psf_stars source, sourceCat source, distance in pixels). 

749 best = {} 

750 for match_psf, match_stars, d in matches: 

751 match = best.get(match_psf.getId()) 

752 if match is None or d <= match[2]: 

753 best[match_psf.getId()] = (match_psf, match_stars, d) 

754 matches = list(best.values()) 

755 # We'll use this to construct index arrays into each catalog. 

756 ids = np.array([(match_psf.getId(), match_stars.getId()) for match_psf, match_stars, d in matches]).T 

757 

758 # Check that no stars sources are listed twice; we already know 

759 # that each match has a unique psf_stars id, due to using as the key 

760 # in best above. 

761 n_matches = len(matches) 

762 n_unique = len(set(m[1].getId() for m in matches)) 

763 if n_unique != n_matches: 

764 self.log.warning("%d psf_stars matched only %d stars; ", 

765 n_matches, n_unique) 

766 if n_matches == 0: 

767 msg = (f"0 psf_stars out of {len(psf_stars)} matched {len(stars)} calib stars." 

768 " Downstream processes probably won't have useful stars in this case." 

769 " Is `star_source_selector` too strict?") 

770 # TODO DM-39842: Turn this into an AlgorithmicError. 

771 raise RuntimeError(msg) 

772 

773 # The indices of the IDs, so we can update the flag fields as arrays. 

774 idx_psf_stars = np.searchsorted(psf_stars["id"], ids[0]) 

775 idx_stars = np.searchsorted(stars["id"], ids[1]) 

776 for field in self.psf_fields: 

777 result = np.zeros(len(stars), dtype=bool) 

778 result[idx_stars] = psf_stars[field][idx_psf_stars] 

779 stars[field] = result 

780 

781 def _fit_astrometry(self, exposure, stars): 

782 """Fit an astrometric model to the data and return the reference 

783 matches used in the fit, and the fitted WCS. 

784 

785 Parameters 

786 ---------- 

787 exposure : `lsst.afw.image.Exposure` 

788 Exposure that is being fit, to get PSF and other metadata from. 

789 Modified to add the fitted skyWcs. 

790 stars : `SourceCatalog` 

791 Good stars selected for use in calibration, with RA/Dec coordinates 

792 computed from the pixel positions and fitted WCS. 

793 

794 Returns 

795 ------- 

796 matches : `list` [`lsst.afw.table.ReferenceMatch`] 

797 Reference/stars matches used in the fit. 

798 """ 

799 result = self.astrometry.run(stars, exposure) 

800 return result.matches, result.matchMeta 

801 

802 def _fit_photometry(self, exposure, stars): 

803 """Fit a photometric model to the data and return the reference 

804 matches used in the fit, and the fitted PhotoCalib. 

805 

806 Parameters 

807 ---------- 

808 exposure : `lsst.afw.image.Exposure` 

809 Exposure that is being fit, to get PSF and other metadata from. 

810 Modified to be in nanojanksy units, with an assigned photoCalib 

811 identically 1. 

812 stars : `lsst.afw.table.SourceCatalog` 

813 Good stars selected for use in calibration. 

814 

815 Returns 

816 ------- 

817 calibrated_stars : `lsst.afw.table.SourceCatalog` 

818 Star catalog with flux/magnitude columns computed from the fitted 

819 photoCalib. 

820 matches : `list` [`lsst.afw.table.ReferenceMatch`] 

821 Reference/stars matches used in the fit. 

822 photoCalib : `lsst.afw.image.PhotoCalib` 

823 Photometric calibration that was fit to the star catalog. 

824 """ 

825 result = self.photometry.run(exposure, stars) 

826 calibrated_stars = result.photoCalib.calibrateCatalog(stars) 

827 exposure.maskedImage = result.photoCalib.calibrateImage(exposure.maskedImage) 

828 identity = afwImage.PhotoCalib(1.0, 

829 result.photoCalib.getCalibrationErr(), 

830 bbox=exposure.getBBox()) 

831 exposure.setPhotoCalib(identity) 

832 

833 return calibrated_stars, result.matches, result.matchMeta, result.photoCalib 

834 

835 def _summarize(self, exposure, stars, background): 

836 """Compute summary statistics on the exposure and update in-place the 

837 calibrations attached to it. 

838 

839 Parameters 

840 ---------- 

841 exposure : `lsst.afw.image.Exposure` 

842 Exposure that was calibrated, to get PSF and other metadata from. 

843 Modified to contain the computed summary statistics. 

844 stars : `SourceCatalog` 

845 Good stars selected used in calibration. 

846 background : `lsst.afw.math.BackgroundList` 

847 Background that was fit to the exposure during detection of the 

848 above stars. 

849 """ 

850 # TODO investigation: because this takes the photoCalib from the 

851 # exposure, photometric summary values may be "incorrect" (i.e. they 

852 # will reflect the ==1 nJy calibration on the exposure, not the 

853 # applied calibration). This needs to be checked. 

854 summary = self.compute_summary_stats.run(exposure, stars, background) 

855 exposure.info.setSummaryStats(summary)