Coverage for python/lsst/pipe/tasks/characterizeImage.py: 31%

180 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 02:18 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"] 

23 

24import numpy as np 

25 

26from lsstDebug import getDebugFrame 

27import lsst.afw.table as afwTable 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.daf.base as dafBase 

31import lsst.pipe.base.connectionTypes as cT 

32from lsst.afw.math import BackgroundList 

33from lsst.afw.table import SourceTable 

34from lsst.meas.algorithms import ( 

35 SubtractBackgroundTask, 

36 SourceDetectionTask, 

37 MeasureApCorrTask, 

38 MeasureApCorrError, 

39 MaskStreaksTask, 

40) 

41from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask 

42from lsst.meas.astrom import displayAstrometry 

43from lsst.meas.base import ( 

44 SingleFrameMeasurementTask, 

45 ApplyApCorrTask, 

46 CatalogCalculationTask, 

47 IdGenerator, 

48 DetectorVisitIdGeneratorConfig, 

49) 

50from lsst.meas.deblender import SourceDeblendTask 

51import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin 

52from .measurePsf import MeasurePsfTask 

53from .repair import RepairTask 

54from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

55from lsst.pex.exceptions import LengthError 

56from lsst.utils.timer import timeMethod 

57 

58 

59class CharacterizeImageConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("instrument", "visit", "detector")): 

61 exposure = cT.Input( 

62 doc="Input exposure data", 

63 name="postISRCCD", 

64 storageClass="Exposure", 

65 dimensions=["instrument", "exposure", "detector"], 

66 ) 

67 characterized = cT.Output( 

68 doc="Output characterized data.", 

69 name="icExp", 

70 storageClass="ExposureF", 

71 dimensions=["instrument", "visit", "detector"], 

72 ) 

73 sourceCat = cT.Output( 

74 doc="Output source catalog.", 

75 name="icSrc", 

76 storageClass="SourceCatalog", 

77 dimensions=["instrument", "visit", "detector"], 

78 ) 

79 backgroundModel = cT.Output( 

80 doc="Output background model.", 

81 name="icExpBackground", 

82 storageClass="Background", 

83 dimensions=["instrument", "visit", "detector"], 

84 ) 

85 outputSchema = cT.InitOutput( 

86 doc="Schema of the catalog produced by CharacterizeImage", 

87 name="icSrc_schema", 

88 storageClass="SourceCatalog", 

89 ) 

90 

91 def adjustQuantum(self, inputs, outputs, label, dataId): 

92 # Docstring inherited from PipelineTaskConnections 

93 try: 

94 return super().adjustQuantum(inputs, outputs, label, dataId) 

95 except pipeBase.ScalarError as err: 

96 raise pipeBase.ScalarError( 

97 "CharacterizeImageTask can at present only be run on visits that are associated with " 

98 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the " 

99 "snap-combination step you probably want hasn't been configured to run between ISR and " 

100 "this task (as of this writing, that would be because it hasn't been implemented yet)." 

101 ) from err 

102 

103 

104class CharacterizeImageConfig(pipeBase.PipelineTaskConfig, 

105 pipelineConnections=CharacterizeImageConnections): 

106 """Config for CharacterizeImageTask.""" 

107 

108 doMeasurePsf = pexConfig.Field( 

109 dtype=bool, 

110 default=True, 

111 doc="Measure PSF? If False then for all subsequent operations use either existing PSF " 

112 "model when present, or install simple PSF model when not (see installSimplePsf " 

113 "config options)" 

114 ) 

115 doWrite = pexConfig.Field( 

116 dtype=bool, 

117 default=True, 

118 doc="Persist results?", 

119 ) 

120 doWriteExposure = pexConfig.Field( 

121 dtype=bool, 

122 default=True, 

123 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.", 

124 ) 

125 psfIterations = pexConfig.RangeField( 

126 dtype=int, 

127 default=2, 

128 min=1, 

129 doc="Number of iterations of detect sources, measure sources, " 

130 "estimate PSF. If useSimplePsf is True then 2 should be plenty; " 

131 "otherwise more may be wanted.", 

132 ) 

133 background = pexConfig.ConfigurableField( 

134 target=SubtractBackgroundTask, 

135 doc="Configuration for initial background estimation", 

136 ) 

137 detection = pexConfig.ConfigurableField( 

138 target=SourceDetectionTask, 

139 doc="Detect sources" 

140 ) 

141 doDeblend = pexConfig.Field( 

142 dtype=bool, 

143 default=True, 

144 doc="Run deblender input exposure" 

145 ) 

146 deblend = pexConfig.ConfigurableField( 

147 target=SourceDeblendTask, 

148 doc="Split blended source into their components" 

149 ) 

150 measurement = pexConfig.ConfigurableField( 

151 target=SingleFrameMeasurementTask, 

152 doc="Measure sources" 

153 ) 

154 doApCorr = pexConfig.Field( 

155 dtype=bool, 

156 default=True, 

157 doc="Run subtasks to measure and apply aperture corrections" 

158 ) 

159 measureApCorr = pexConfig.ConfigurableField( 

160 target=MeasureApCorrTask, 

161 doc="Subtask to measure aperture corrections" 

162 ) 

163 applyApCorr = pexConfig.ConfigurableField( 

164 target=ApplyApCorrTask, 

165 doc="Subtask to apply aperture corrections" 

166 ) 

167 # If doApCorr is False, and the exposure does not have apcorrections already applied, the 

168 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin 

169 catalogCalculation = pexConfig.ConfigurableField( 

170 target=CatalogCalculationTask, 

171 doc="Subtask to run catalogCalculation plugins on catalog" 

172 ) 

173 doComputeSummaryStats = pexConfig.Field( 

174 dtype=bool, 

175 default=True, 

176 doc="Run subtask to measure exposure summary statistics", 

177 deprecated=("This subtask has been moved to CalibrateTask " 

178 "with DM-30701.") 

179 ) 

180 computeSummaryStats = pexConfig.ConfigurableField( 

181 target=ComputeExposureSummaryStatsTask, 

182 doc="Subtask to run computeSummaryStats on exposure", 

183 deprecated=("This subtask has been moved to CalibrateTask " 

184 "with DM-30701.") 

185 ) 

186 useSimplePsf = pexConfig.Field( 

187 dtype=bool, 

188 default=True, 

189 doc="Replace the existing PSF model with a simplified version that has the same sigma " 

190 "at the start of each PSF determination iteration? Doing so makes PSF determination " 

191 "converge more robustly and quickly.", 

192 ) 

193 installSimplePsf = pexConfig.ConfigurableField( 

194 target=InstallGaussianPsfTask, 

195 doc="Install a simple PSF model", 

196 ) 

197 measurePsf = pexConfig.ConfigurableField( 

198 target=MeasurePsfTask, 

199 doc="Measure PSF", 

200 ) 

201 repair = pexConfig.ConfigurableField( 

202 target=RepairTask, 

203 doc="Remove cosmic rays", 

204 ) 

205 requireCrForPsf = pexConfig.Field( 

206 dtype=bool, 

207 default=True, 

208 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF." 

209 ) 

210 checkUnitsParseStrict = pexConfig.Field( 

211 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'", 

212 dtype=str, 

213 default="raise", 

214 ) 

215 doMaskStreaks = pexConfig.Field( 

216 doc="Mask streaks", 

217 default=True, 

218 dtype=bool, 

219 ) 

220 maskStreaks = pexConfig.ConfigurableField( 

221 target=MaskStreaksTask, 

222 doc="Subtask for masking streaks. Only used if doMaskStreaks is True. " 

223 "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.", 

224 ) 

225 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

226 

227 def setDefaults(self): 

228 super().setDefaults() 

229 # Just detect bright stars. 

230 # The thresholdValue sets the minimum flux in a pixel to be included in the 

231 # footprint, while peaks are only detected when they are above 

232 # thresholdValue * includeThresholdMultiplier. The low thresholdValue 

233 # ensures that the footprints are large enough for the noise replacer 

234 # to mask out faint undetected neighbors that are not to be measured. 

235 self.detection.thresholdValue = 5.0 

236 self.detection.includeThresholdMultiplier = 10.0 

237 # do not deblend, as it makes a mess 

238 self.doDeblend = False 

239 # measure and apply aperture correction; note: measuring and applying aperture 

240 # correction are disabled until the final measurement, after PSF is measured 

241 self.doApCorr = True 

242 # During characterization, we don't have full source measurement information, 

243 # so must do the aperture correction with only psf stars, combined with the 

244 # default signal-to-noise cuts in MeasureApCorrTask. 

245 selector = self.measureApCorr.sourceSelector["science"] 

246 selector.doUnresolved = False 

247 selector.flags.good = ["calib_psf_used"] 

248 selector.flags.bad = [] 

249 

250 # minimal set of measurements needed to determine PSF 

251 self.measurement.plugins.names = [ 

252 "base_PixelFlags", 

253 "base_SdssCentroid", 

254 "ext_shapeHSM_HsmSourceMoments", 

255 "base_GaussianFlux", 

256 "base_PsfFlux", 

257 "base_CircularApertureFlux", 

258 "base_ClassificationSizeExtendedness", 

259 ] 

260 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

261 

262 def validate(self): 

263 if self.doApCorr and not self.measurePsf: 

264 raise RuntimeError("Must measure PSF to measure aperture correction, " 

265 "because flags determined by PSF measurement are used to identify " 

266 "sources used to measure aperture correction") 

267 

268 

269class CharacterizeImageTask(pipeBase.PipelineTask): 

270 """Measure bright sources and use this to estimate background and PSF of 

271 an exposure. 

272 

273 Given an exposure with defects repaired (masked and interpolated over, 

274 e.g. as output by `~lsst.ip.isr.IsrTask`): 

275 - detect and measure bright sources 

276 - repair cosmic rays 

277 - detect and mask streaks 

278 - measure and subtract background 

279 - measure PSF 

280 

281 Parameters 

282 ---------- 

283 schema : `lsst.afw.table.Schema`, optional 

284 Initial schema for icSrc catalog. 

285 **kwargs 

286 Additional keyword arguments. 

287 

288 Notes 

289 ----- 

290 Debugging: 

291 CharacterizeImageTask has a debug dictionary with the following keys: 

292 

293 frame 

294 int: if specified, the frame of first debug image displayed (defaults to 1) 

295 repair_iter 

296 bool; if True display image after each repair in the measure PSF loop 

297 background_iter 

298 bool; if True display image after each background subtraction in the measure PSF loop 

299 measure_iter 

300 bool; if True display image and sources at the end of each iteration of the measure PSF loop 

301 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols. 

302 psf 

303 bool; if True display image and sources after PSF is measured; 

304 this will be identical to the final image displayed by measure_iter if measure_iter is true 

305 repair 

306 bool; if True display image and sources after final repair 

307 measure 

308 bool; if True display image and sources after final measurement 

309 """ 

310 

311 ConfigClass = CharacterizeImageConfig 

312 _DefaultName = "characterizeImage" 

313 

314 def __init__(self, schema=None, **kwargs): 

315 super().__init__(**kwargs) 

316 

317 if schema is None: 

318 schema = SourceTable.makeMinimalSchema() 

319 self.schema = schema 

320 self.makeSubtask("background") 

321 self.makeSubtask("installSimplePsf") 

322 self.makeSubtask("repair") 

323 if self.config.doMaskStreaks: 

324 self.makeSubtask("maskStreaks") 

325 self.makeSubtask("measurePsf", schema=self.schema) 

326 self.algMetadata = dafBase.PropertyList() 

327 self.makeSubtask('detection', schema=self.schema) 

328 if self.config.doDeblend: 

329 self.makeSubtask("deblend", schema=self.schema) 

330 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata) 

331 if self.config.doApCorr: 

332 self.makeSubtask('measureApCorr', schema=self.schema) 

333 self.makeSubtask('applyApCorr', schema=self.schema) 

334 self.makeSubtask('catalogCalculation', schema=self.schema) 

335 self._initialFrame = getDebugFrame(self._display, "frame") or 1 

336 self._frame = self._initialFrame 

337 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

338 afwTable.CoordKey.addErrorFields(self.schema) 

339 self.outputSchema = afwTable.SourceCatalog(self.schema) 

340 

341 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

342 inputs = butlerQC.get(inputRefs) 

343 if 'idGenerator' not in inputs.keys(): 

344 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

345 outputs = self.run(**inputs) 

346 butlerQC.put(outputs, outputRefs) 

347 

348 @timeMethod 

349 def run(self, exposure, background=None, idGenerator=None): 

350 """Characterize a science image. 

351 

352 Peforms the following operations: 

353 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: 

354 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) 

355 - interpolate over cosmic rays 

356 - perform final measurement 

357 

358 Parameters 

359 ---------- 

360 exposure : `lsst.afw.image.ExposureF` 

361 Exposure to characterize. 

362 background : `lsst.afw.math.BackgroundList`, optional 

363 Initial model of background already subtracted from exposure. 

364 idGenerator : `lsst.meas.base.IdGenerator`, optional 

365 Object that generates source IDs and provides RNG seeds. 

366 

367 Returns 

368 ------- 

369 result : `lsst.pipe.base.Struct` 

370 Results as a struct with attributes: 

371 

372 ``exposure`` 

373 Characterized exposure (`lsst.afw.image.ExposureF`). 

374 ``sourceCat`` 

375 Detected sources (`lsst.afw.table.SourceCatalog`). 

376 ``background`` 

377 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

378 ``psfCellSet`` 

379 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

380 ``characterized`` 

381 Another reference to ``exposure`` for compatibility. 

382 ``backgroundModel`` 

383 Another reference to ``background`` for compatibility. 

384 

385 Raises 

386 ------ 

387 RuntimeError 

388 Raised if PSF sigma is NaN. 

389 """ 

390 self._frame = self._initialFrame # reset debug display frame 

391 

392 if not self.config.doMeasurePsf and not exposure.hasPsf(): 

393 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.") 

394 self.installSimplePsf.run(exposure=exposure) 

395 

396 if idGenerator is None: 

397 idGenerator = IdGenerator() 

398 

399 # subtract an initial estimate of background level 

400 background = self.background.run(exposure).background 

401 

402 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1 

403 for i in range(psfIterations): 

404 dmeRes = self.detectMeasureAndEstimatePsf( 

405 exposure=exposure, 

406 idGenerator=idGenerator, 

407 background=background, 

408 ) 

409 

410 psf = dmeRes.exposure.getPsf() 

411 # Just need a rough estimate; average positions are fine 

412 psfAvgPos = psf.getAveragePosition() 

413 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius() 

414 psfDimensions = psf.computeImage(psfAvgPos).getDimensions() 

415 medBackground = np.median(dmeRes.background.getImage().getArray()) 

416 self.log.info("iter %s; PSF sigma=%0.4f, dimensions=%s; median background=%0.2f", 

417 i + 1, psfSigma, psfDimensions, medBackground) 

418 if np.isnan(psfSigma): 

419 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.") 

420 

421 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

422 

423 # perform final repair with final PSF 

424 self.repair.run(exposure=dmeRes.exposure) 

425 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

426 

427 # mask streaks 

428 if self.config.doMaskStreaks: 

429 _ = self.maskStreaks.run(dmeRes.exposure) 

430 

431 # perform final measurement with final PSF, including measuring and applying aperture correction, 

432 # if wanted 

433 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, 

434 exposureId=idGenerator.catalog_id) 

435 if self.config.doApCorr: 

436 try: 

437 apCorrMap = self.measureApCorr.run( 

438 exposure=dmeRes.exposure, 

439 catalog=dmeRes.sourceCat, 

440 ).apCorrMap 

441 except MeasureApCorrError: 

442 # We have failed to get a valid aperture correction map. 

443 # Proceed with processing, and image will be filtered 

444 # downstream. 

445 dmeRes.exposure.info.setApCorrMap(None) 

446 else: 

447 dmeRes.exposure.info.setApCorrMap(apCorrMap) 

448 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) 

449 

450 self.catalogCalculation.run(dmeRes.sourceCat) 

451 

452 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

453 

454 return pipeBase.Struct( 

455 exposure=dmeRes.exposure, 

456 sourceCat=dmeRes.sourceCat, 

457 background=dmeRes.background, 

458 psfCellSet=dmeRes.psfCellSet, 

459 

460 characterized=dmeRes.exposure, 

461 backgroundModel=dmeRes.background 

462 ) 

463 

464 @timeMethod 

465 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background): 

466 """Perform one iteration of detect, measure, and estimate PSF. 

467 

468 Performs the following operations: 

469 

470 - if config.doMeasurePsf or not exposure.hasPsf(): 

471 

472 - install a simple PSF model (replacing the existing one, if need be) 

473 

474 - interpolate over cosmic rays with keepCRs=True 

475 - estimate background and subtract it from the exposure 

476 - detect, deblend and measure sources, and subtract a refined background model; 

477 - if config.doMeasurePsf: 

478 - measure PSF 

479 

480 Parameters 

481 ---------- 

482 exposure : `lsst.afw.image.ExposureF` 

483 Exposure to characterize. 

484 idGenerator : `lsst.meas.base.IdGenerator` 

485 Object that generates source IDs and provides RNG seeds. 

486 background : `lsst.afw.math.BackgroundList`, optional 

487 Initial model of background already subtracted from exposure. 

488 

489 Returns 

490 ------- 

491 result : `lsst.pipe.base.Struct` 

492 Results as a struct with attributes: 

493 

494 ``exposure`` 

495 Characterized exposure (`lsst.afw.image.ExposureF`). 

496 ``sourceCat`` 

497 Detected sources (`lsst.afw.table.SourceCatalog`). 

498 ``background`` 

499 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

500 ``psfCellSet`` 

501 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

502 

503 Raises 

504 ------ 

505 LengthError 

506 Raised if there are too many CR pixels. 

507 """ 

508 # install a simple PSF model, if needed or wanted 

509 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): 

510 self.log.info("PSF estimation initialized with 'simple' PSF") 

511 self.installSimplePsf.run(exposure=exposure) 

512 

513 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model) 

514 if self.config.requireCrForPsf: 

515 self.repair.run(exposure=exposure, keepCRs=True) 

516 else: 

517 try: 

518 self.repair.run(exposure=exposure, keepCRs=True) 

519 except LengthError: 

520 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)", 

521 self.config.repair.cosmicray.nCrPixelMax) 

522 

523 self.display("repair_iter", exposure=exposure) 

524 

525 if background is None: 

526 background = BackgroundList() 

527 

528 sourceIdFactory = idGenerator.make_table_id_factory() 

529 table = SourceTable.make(self.schema, sourceIdFactory) 

530 table.setMetadata(self.algMetadata) 

531 

532 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) 

533 sourceCat = detRes.sources 

534 if detRes.background: 

535 for bg in detRes.background: 

536 background.append(bg) 

537 

538 if self.config.doDeblend: 

539 self.deblend.run(exposure=exposure, sources=sourceCat) 

540 # We need the output catalog to be contiguous for further processing. 

541 if not sourceCat.isContiguous(): 

542 sourceCat = sourceCat.copy(deep=True) 

543 

544 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id) 

545 

546 measPsfRes = pipeBase.Struct(cellSet=None) 

547 if self.config.doMeasurePsf: 

548 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, 

549 expId=idGenerator.catalog_id) 

550 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat) 

551 

552 return pipeBase.Struct( 

553 exposure=exposure, 

554 sourceCat=sourceCat, 

555 background=background, 

556 psfCellSet=measPsfRes.cellSet, 

557 ) 

558 

559 def display(self, itemName, exposure, sourceCat=None): 

560 """Display exposure and sources on next frame (for debugging). 

561 

562 Parameters 

563 ---------- 

564 itemName : `str` 

565 Name of item in ``debugInfo``. 

566 exposure : `lsst.afw.image.ExposureF` 

567 Exposure to display. 

568 sourceCat : `lsst.afw.table.SourceCatalog`, optional 

569 Catalog of sources detected on the exposure. 

570 """ 

571 val = getDebugFrame(self._display, itemName) 

572 if not val: 

573 return 

574 

575 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False) 

576 self._frame += 1