Coverage for python/lsst/pipe/tasks/characterizeImage.py: 31%

180 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-12 10:42 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"] 

23 

24import numpy as np 

25 

26from lsstDebug import getDebugFrame 

27import lsst.afw.table as afwTable 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.daf.base as dafBase 

31import lsst.pipe.base.connectionTypes as cT 

32from lsst.afw.math import BackgroundList 

33from lsst.afw.table import SourceTable 

34from lsst.meas.algorithms import ( 

35 SubtractBackgroundTask, 

36 SourceDetectionTask, 

37 MeasureApCorrTask, 

38 MeasureApCorrError, 

39) 

40from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask 

41from lsst.meas.astrom import displayAstrometry 

42from lsst.meas.base import ( 

43 SingleFrameMeasurementTask, 

44 ApplyApCorrTask, 

45 CatalogCalculationTask, 

46 IdGenerator, 

47 DetectorVisitIdGeneratorConfig, 

48) 

49from lsst.meas.deblender import SourceDeblendTask 

50import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin 

51from .measurePsf import MeasurePsfTask 

52from .repair import RepairTask 

53from .maskStreaks import MaskStreaksTask 

54from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

55from lsst.pex.exceptions import LengthError 

56from lsst.utils.timer import timeMethod 

57 

58 

59class CharacterizeImageConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("instrument", "visit", "detector")): 

61 exposure = cT.Input( 

62 doc="Input exposure data", 

63 name="postISRCCD", 

64 storageClass="Exposure", 

65 dimensions=["instrument", "exposure", "detector"], 

66 ) 

67 characterized = cT.Output( 

68 doc="Output characterized data.", 

69 name="icExp", 

70 storageClass="ExposureF", 

71 dimensions=["instrument", "visit", "detector"], 

72 ) 

73 sourceCat = cT.Output( 

74 doc="Output source catalog.", 

75 name="icSrc", 

76 storageClass="SourceCatalog", 

77 dimensions=["instrument", "visit", "detector"], 

78 ) 

79 backgroundModel = cT.Output( 

80 doc="Output background model.", 

81 name="icExpBackground", 

82 storageClass="Background", 

83 dimensions=["instrument", "visit", "detector"], 

84 ) 

85 outputSchema = cT.InitOutput( 

86 doc="Schema of the catalog produced by CharacterizeImage", 

87 name="icSrc_schema", 

88 storageClass="SourceCatalog", 

89 ) 

90 

91 def adjustQuantum(self, inputs, outputs, label, dataId): 

92 # Docstring inherited from PipelineTaskConnections 

93 try: 

94 return super().adjustQuantum(inputs, outputs, label, dataId) 

95 except pipeBase.ScalarError as err: 

96 raise pipeBase.ScalarError( 

97 "CharacterizeImageTask can at present only be run on visits that are associated with " 

98 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the " 

99 "snap-combination step you probably want hasn't been configured to run between ISR and " 

100 "this task (as of this writing, that would be because it hasn't been implemented yet)." 

101 ) from err 

102 

103 

104class CharacterizeImageConfig(pipeBase.PipelineTaskConfig, 

105 pipelineConnections=CharacterizeImageConnections): 

106 """Config for CharacterizeImageTask.""" 

107 

108 doMeasurePsf = pexConfig.Field( 

109 dtype=bool, 

110 default=True, 

111 doc="Measure PSF? If False then for all subsequent operations use either existing PSF " 

112 "model when present, or install simple PSF model when not (see installSimplePsf " 

113 "config options)" 

114 ) 

115 doWrite = pexConfig.Field( 

116 dtype=bool, 

117 default=True, 

118 doc="Persist results?", 

119 ) 

120 doWriteExposure = pexConfig.Field( 

121 dtype=bool, 

122 default=True, 

123 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.", 

124 ) 

125 psfIterations = pexConfig.RangeField( 

126 dtype=int, 

127 default=2, 

128 min=1, 

129 doc="Number of iterations of detect sources, measure sources, " 

130 "estimate PSF. If useSimplePsf is True then 2 should be plenty; " 

131 "otherwise more may be wanted.", 

132 ) 

133 background = pexConfig.ConfigurableField( 

134 target=SubtractBackgroundTask, 

135 doc="Configuration for initial background estimation", 

136 ) 

137 detection = pexConfig.ConfigurableField( 

138 target=SourceDetectionTask, 

139 doc="Detect sources" 

140 ) 

141 doDeblend = pexConfig.Field( 

142 dtype=bool, 

143 default=True, 

144 doc="Run deblender input exposure" 

145 ) 

146 deblend = pexConfig.ConfigurableField( 

147 target=SourceDeblendTask, 

148 doc="Split blended source into their components" 

149 ) 

150 measurement = pexConfig.ConfigurableField( 

151 target=SingleFrameMeasurementTask, 

152 doc="Measure sources" 

153 ) 

154 doApCorr = pexConfig.Field( 

155 dtype=bool, 

156 default=True, 

157 doc="Run subtasks to measure and apply aperture corrections" 

158 ) 

159 measureApCorr = pexConfig.ConfigurableField( 

160 target=MeasureApCorrTask, 

161 doc="Subtask to measure aperture corrections" 

162 ) 

163 applyApCorr = pexConfig.ConfigurableField( 

164 target=ApplyApCorrTask, 

165 doc="Subtask to apply aperture corrections" 

166 ) 

167 # If doApCorr is False, and the exposure does not have apcorrections already applied, the 

168 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin 

169 catalogCalculation = pexConfig.ConfigurableField( 

170 target=CatalogCalculationTask, 

171 doc="Subtask to run catalogCalculation plugins on catalog" 

172 ) 

173 doComputeSummaryStats = pexConfig.Field( 

174 dtype=bool, 

175 default=True, 

176 doc="Run subtask to measure exposure summary statistics", 

177 deprecated=("This subtask has been moved to CalibrateTask " 

178 "with DM-30701.") 

179 ) 

180 computeSummaryStats = pexConfig.ConfigurableField( 

181 target=ComputeExposureSummaryStatsTask, 

182 doc="Subtask to run computeSummaryStats on exposure", 

183 deprecated=("This subtask has been moved to CalibrateTask " 

184 "with DM-30701.") 

185 ) 

186 useSimplePsf = pexConfig.Field( 

187 dtype=bool, 

188 default=True, 

189 doc="Replace the existing PSF model with a simplified version that has the same sigma " 

190 "at the start of each PSF determination iteration? Doing so makes PSF determination " 

191 "converge more robustly and quickly.", 

192 ) 

193 installSimplePsf = pexConfig.ConfigurableField( 

194 target=InstallGaussianPsfTask, 

195 doc="Install a simple PSF model", 

196 ) 

197 measurePsf = pexConfig.ConfigurableField( 

198 target=MeasurePsfTask, 

199 doc="Measure PSF", 

200 ) 

201 repair = pexConfig.ConfigurableField( 

202 target=RepairTask, 

203 doc="Remove cosmic rays", 

204 ) 

205 requireCrForPsf = pexConfig.Field( 

206 dtype=bool, 

207 default=True, 

208 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF." 

209 ) 

210 checkUnitsParseStrict = pexConfig.Field( 

211 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'", 

212 dtype=str, 

213 default="raise", 

214 ) 

215 doMaskStreaks = pexConfig.Field( 

216 doc="Mask streaks", 

217 default=True, 

218 dtype=bool, 

219 ) 

220 maskStreaks = pexConfig.ConfigurableField( 

221 target=MaskStreaksTask, 

222 doc="Subtask for masking streaks. Only used if doMaskStreaks is True. " 

223 "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.", 

224 ) 

225 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

226 

227 def setDefaults(self): 

228 super().setDefaults() 

229 # just detect bright stars; includeThresholdMultipler=10 seems large, 

230 # but these are the values we have been using 

231 self.detection.thresholdValue = 5.0 

232 self.detection.includeThresholdMultiplier = 10.0 

233 # do not deblend, as it makes a mess 

234 self.doDeblend = False 

235 # measure and apply aperture correction; note: measuring and applying aperture 

236 # correction are disabled until the final measurement, after PSF is measured 

237 self.doApCorr = True 

238 # During characterization, we don't have full source measurement information, 

239 # so must do the aperture correction with only psf stars, combined with the 

240 # default signal-to-noise cuts in MeasureApCorrTask. 

241 selector = self.measureApCorr.sourceSelector["science"] 

242 selector.doUnresolved = False 

243 selector.flags.good = ["calib_psf_used"] 

244 selector.flags.bad = [] 

245 

246 # minimal set of measurements needed to determine PSF 

247 self.measurement.plugins.names = [ 

248 "base_PixelFlags", 

249 "base_SdssCentroid", 

250 "ext_shapeHSM_HsmSourceMoments", 

251 "base_GaussianFlux", 

252 "base_PsfFlux", 

253 "base_CircularApertureFlux", 

254 ] 

255 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

256 

257 def validate(self): 

258 if self.doApCorr and not self.measurePsf: 

259 raise RuntimeError("Must measure PSF to measure aperture correction, " 

260 "because flags determined by PSF measurement are used to identify " 

261 "sources used to measure aperture correction") 

262 

263 

264class CharacterizeImageTask(pipeBase.PipelineTask): 

265 """Measure bright sources and use this to estimate background and PSF of 

266 an exposure. 

267 

268 Given an exposure with defects repaired (masked and interpolated over, 

269 e.g. as output by `~lsst.ip.isr.IsrTask`): 

270 - detect and measure bright sources 

271 - repair cosmic rays 

272 - detect and mask streaks 

273 - measure and subtract background 

274 - measure PSF 

275 

276 Parameters 

277 ---------- 

278 schema : `lsst.afw.table.Schema`, optional 

279 Initial schema for icSrc catalog. 

280 **kwargs 

281 Additional keyword arguments. 

282 

283 Notes 

284 ----- 

285 Debugging: 

286 CharacterizeImageTask has a debug dictionary with the following keys: 

287 

288 frame 

289 int: if specified, the frame of first debug image displayed (defaults to 1) 

290 repair_iter 

291 bool; if True display image after each repair in the measure PSF loop 

292 background_iter 

293 bool; if True display image after each background subtraction in the measure PSF loop 

294 measure_iter 

295 bool; if True display image and sources at the end of each iteration of the measure PSF loop 

296 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols. 

297 psf 

298 bool; if True display image and sources after PSF is measured; 

299 this will be identical to the final image displayed by measure_iter if measure_iter is true 

300 repair 

301 bool; if True display image and sources after final repair 

302 measure 

303 bool; if True display image and sources after final measurement 

304 """ 

305 

306 ConfigClass = CharacterizeImageConfig 

307 _DefaultName = "characterizeImage" 

308 

309 def __init__(self, schema=None, **kwargs): 

310 super().__init__(**kwargs) 

311 

312 if schema is None: 

313 schema = SourceTable.makeMinimalSchema() 

314 self.schema = schema 

315 self.makeSubtask("background") 

316 self.makeSubtask("installSimplePsf") 

317 self.makeSubtask("repair") 

318 if self.config.doMaskStreaks: 

319 self.makeSubtask("maskStreaks") 

320 self.makeSubtask("measurePsf", schema=self.schema) 

321 self.algMetadata = dafBase.PropertyList() 

322 self.makeSubtask('detection', schema=self.schema) 

323 if self.config.doDeblend: 

324 self.makeSubtask("deblend", schema=self.schema) 

325 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata) 

326 if self.config.doApCorr: 

327 self.makeSubtask('measureApCorr', schema=self.schema) 

328 self.makeSubtask('applyApCorr', schema=self.schema) 

329 self.makeSubtask('catalogCalculation', schema=self.schema) 

330 self._initialFrame = getDebugFrame(self._display, "frame") or 1 

331 self._frame = self._initialFrame 

332 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

333 self.outputSchema = afwTable.SourceCatalog(self.schema) 

334 

335 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

336 inputs = butlerQC.get(inputRefs) 

337 if 'idGenerator' not in inputs.keys(): 

338 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

339 outputs = self.run(**inputs) 

340 butlerQC.put(outputs, outputRefs) 

341 

342 @timeMethod 

343 def run(self, exposure, background=None, idGenerator=None): 

344 """Characterize a science image. 

345 

346 Peforms the following operations: 

347 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: 

348 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) 

349 - interpolate over cosmic rays 

350 - perform final measurement 

351 

352 Parameters 

353 ---------- 

354 exposure : `lsst.afw.image.ExposureF` 

355 Exposure to characterize. 

356 background : `lsst.afw.math.BackgroundList`, optional 

357 Initial model of background already subtracted from exposure. 

358 idGenerator : `lsst.meas.base.IdGenerator`, optional 

359 Object that generates source IDs and provides RNG seeds. 

360 

361 Returns 

362 ------- 

363 result : `lsst.pipe.base.Struct` 

364 Results as a struct with attributes: 

365 

366 ``exposure`` 

367 Characterized exposure (`lsst.afw.image.ExposureF`). 

368 ``sourceCat`` 

369 Detected sources (`lsst.afw.table.SourceCatalog`). 

370 ``background`` 

371 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

372 ``psfCellSet`` 

373 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

374 ``characterized`` 

375 Another reference to ``exposure`` for compatibility. 

376 ``backgroundModel`` 

377 Another reference to ``background`` for compatibility. 

378 

379 Raises 

380 ------ 

381 RuntimeError 

382 Raised if PSF sigma is NaN. 

383 """ 

384 self._frame = self._initialFrame # reset debug display frame 

385 

386 if not self.config.doMeasurePsf and not exposure.hasPsf(): 

387 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.") 

388 self.installSimplePsf.run(exposure=exposure) 

389 

390 if idGenerator is None: 

391 idGenerator = IdGenerator() 

392 

393 # subtract an initial estimate of background level 

394 background = self.background.run(exposure).background 

395 

396 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1 

397 for i in range(psfIterations): 

398 dmeRes = self.detectMeasureAndEstimatePsf( 

399 exposure=exposure, 

400 idGenerator=idGenerator, 

401 background=background, 

402 ) 

403 

404 psf = dmeRes.exposure.getPsf() 

405 # Just need a rough estimate; average positions are fine 

406 psfAvgPos = psf.getAveragePosition() 

407 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius() 

408 psfDimensions = psf.computeImage(psfAvgPos).getDimensions() 

409 medBackground = np.median(dmeRes.background.getImage().getArray()) 

410 self.log.info("iter %s; PSF sigma=%0.4f, dimensions=%s; median background=%0.2f", 

411 i + 1, psfSigma, psfDimensions, medBackground) 

412 if np.isnan(psfSigma): 

413 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.") 

414 

415 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

416 

417 # perform final repair with final PSF 

418 self.repair.run(exposure=dmeRes.exposure) 

419 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

420 

421 # mask streaks 

422 if self.config.doMaskStreaks: 

423 _ = self.maskStreaks.run(dmeRes.exposure) 

424 

425 # perform final measurement with final PSF, including measuring and applying aperture correction, 

426 # if wanted 

427 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, 

428 exposureId=idGenerator.catalog_id) 

429 if self.config.doApCorr: 

430 try: 

431 apCorrMap = self.measureApCorr.run( 

432 exposure=dmeRes.exposure, 

433 catalog=dmeRes.sourceCat, 

434 ).apCorrMap 

435 except MeasureApCorrError: 

436 # We have failed to get a valid aperture correction map. 

437 # Proceed with processing, and image will be filtered 

438 # downstream. 

439 dmeRes.exposure.info.setApCorrMap(None) 

440 else: 

441 dmeRes.exposure.info.setApCorrMap(apCorrMap) 

442 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) 

443 

444 self.catalogCalculation.run(dmeRes.sourceCat) 

445 

446 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

447 

448 return pipeBase.Struct( 

449 exposure=dmeRes.exposure, 

450 sourceCat=dmeRes.sourceCat, 

451 background=dmeRes.background, 

452 psfCellSet=dmeRes.psfCellSet, 

453 

454 characterized=dmeRes.exposure, 

455 backgroundModel=dmeRes.background 

456 ) 

457 

458 @timeMethod 

459 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background): 

460 """Perform one iteration of detect, measure, and estimate PSF. 

461 

462 Performs the following operations: 

463 

464 - if config.doMeasurePsf or not exposure.hasPsf(): 

465 

466 - install a simple PSF model (replacing the existing one, if need be) 

467 

468 - interpolate over cosmic rays with keepCRs=True 

469 - estimate background and subtract it from the exposure 

470 - detect, deblend and measure sources, and subtract a refined background model; 

471 - if config.doMeasurePsf: 

472 - measure PSF 

473 

474 Parameters 

475 ---------- 

476 exposure : `lsst.afw.image.ExposureF` 

477 Exposure to characterize. 

478 idGenerator : `lsst.meas.base.IdGenerator` 

479 Object that generates source IDs and provides RNG seeds. 

480 background : `lsst.afw.math.BackgroundList`, optional 

481 Initial model of background already subtracted from exposure. 

482 

483 Returns 

484 ------- 

485 result : `lsst.pipe.base.Struct` 

486 Results as a struct with attributes: 

487 

488 ``exposure`` 

489 Characterized exposure (`lsst.afw.image.ExposureF`). 

490 ``sourceCat`` 

491 Detected sources (`lsst.afw.table.SourceCatalog`). 

492 ``background`` 

493 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

494 ``psfCellSet`` 

495 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

496 

497 Raises 

498 ------ 

499 LengthError 

500 Raised if there are too many CR pixels. 

501 """ 

502 # install a simple PSF model, if needed or wanted 

503 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): 

504 self.log.info("PSF estimation initialized with 'simple' PSF") 

505 self.installSimplePsf.run(exposure=exposure) 

506 

507 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model) 

508 if self.config.requireCrForPsf: 

509 self.repair.run(exposure=exposure, keepCRs=True) 

510 else: 

511 try: 

512 self.repair.run(exposure=exposure, keepCRs=True) 

513 except LengthError: 

514 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)", 

515 self.config.repair.cosmicray.nCrPixelMax) 

516 

517 self.display("repair_iter", exposure=exposure) 

518 

519 if background is None: 

520 background = BackgroundList() 

521 

522 sourceIdFactory = idGenerator.make_table_id_factory() 

523 table = SourceTable.make(self.schema, sourceIdFactory) 

524 table.setMetadata(self.algMetadata) 

525 

526 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) 

527 sourceCat = detRes.sources 

528 if detRes.background: 

529 for bg in detRes.background: 

530 background.append(bg) 

531 

532 if self.config.doDeblend: 

533 self.deblend.run(exposure=exposure, sources=sourceCat) 

534 # We need the output catalog to be contiguous for further processing. 

535 if not sourceCat.isContiguous(): 

536 sourceCat = sourceCat.copy(deep=True) 

537 

538 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id) 

539 

540 measPsfRes = pipeBase.Struct(cellSet=None) 

541 if self.config.doMeasurePsf: 

542 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, 

543 expId=idGenerator.catalog_id) 

544 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat) 

545 

546 return pipeBase.Struct( 

547 exposure=exposure, 

548 sourceCat=sourceCat, 

549 background=background, 

550 psfCellSet=measPsfRes.cellSet, 

551 ) 

552 

553 def display(self, itemName, exposure, sourceCat=None): 

554 """Display exposure and sources on next frame (for debugging). 

555 

556 Parameters 

557 ---------- 

558 itemName : `str` 

559 Name of item in ``debugInfo``. 

560 exposure : `lsst.afw.image.ExposureF` 

561 Exposure to display. 

562 sourceCat : `lsst.afw.table.SourceCatalog`, optional 

563 Catalog of sources detected on the exposure. 

564 """ 

565 val = getDebugFrame(self._display, itemName) 

566 if not val: 

567 return 

568 

569 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False) 

570 self._frame += 1