Coverage for python/lsst/pipe/tasks/characterizeImage.py: 30%

184 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-22 03:00 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"] 

23 

24import numpy as np 

25import warnings 

26 

27from lsstDebug import getDebugFrame 

28import lsst.afw.table as afwTable 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.daf.base as dafBase 

32import lsst.pipe.base.connectionTypes as cT 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import ( 

36 SubtractBackgroundTask, 

37 SourceDetectionTask, 

38 MeasureApCorrTask, 

39 MeasureApCorrError, 

40) 

41from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask 

42from lsst.meas.astrom import RefMatchTask, displayAstrometry 

43from lsst.meas.algorithms import LoadReferenceObjectsConfig 

44from lsst.obs.base import ExposureIdInfo 

45from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask 

46from lsst.meas.deblender import SourceDeblendTask 

47import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin 

48from .measurePsf import MeasurePsfTask 

49from .repair import RepairTask 

50from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

51from lsst.pex.exceptions import LengthError 

52from lsst.utils.timer import timeMethod 

53 

54 

55class CharacterizeImageConnections(pipeBase.PipelineTaskConnections, 

56 dimensions=("instrument", "visit", "detector")): 

57 exposure = cT.Input( 

58 doc="Input exposure data", 

59 name="postISRCCD", 

60 storageClass="Exposure", 

61 dimensions=["instrument", "exposure", "detector"], 

62 ) 

63 characterized = cT.Output( 

64 doc="Output characterized data.", 

65 name="icExp", 

66 storageClass="ExposureF", 

67 dimensions=["instrument", "visit", "detector"], 

68 ) 

69 sourceCat = cT.Output( 

70 doc="Output source catalog.", 

71 name="icSrc", 

72 storageClass="SourceCatalog", 

73 dimensions=["instrument", "visit", "detector"], 

74 ) 

75 backgroundModel = cT.Output( 

76 doc="Output background model.", 

77 name="icExpBackground", 

78 storageClass="Background", 

79 dimensions=["instrument", "visit", "detector"], 

80 ) 

81 outputSchema = cT.InitOutput( 

82 doc="Schema of the catalog produced by CharacterizeImage", 

83 name="icSrc_schema", 

84 storageClass="SourceCatalog", 

85 ) 

86 

87 def adjustQuantum(self, inputs, outputs, label, dataId): 

88 # Docstring inherited from PipelineTaskConnections 

89 try: 

90 return super().adjustQuantum(inputs, outputs, label, dataId) 

91 except pipeBase.ScalarError as err: 

92 raise pipeBase.ScalarError( 

93 "CharacterizeImageTask can at present only be run on visits that are associated with " 

94 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the " 

95 "snap-combination step you probably want hasn't been configured to run between ISR and " 

96 "this task (as of this writing, that would be because it hasn't been implemented yet)." 

97 ) from err 

98 

99 

100class CharacterizeImageConfig(pipeBase.PipelineTaskConfig, 

101 pipelineConnections=CharacterizeImageConnections): 

102 """Config for CharacterizeImageTask.""" 

103 

104 doMeasurePsf = pexConfig.Field( 

105 dtype=bool, 

106 default=True, 

107 doc="Measure PSF? If False then for all subsequent operations use either existing PSF " 

108 "model when present, or install simple PSF model when not (see installSimplePsf " 

109 "config options)" 

110 ) 

111 doWrite = pexConfig.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Persist results?", 

115 ) 

116 doWriteExposure = pexConfig.Field( 

117 dtype=bool, 

118 default=True, 

119 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.", 

120 ) 

121 psfIterations = pexConfig.RangeField( 

122 dtype=int, 

123 default=2, 

124 min=1, 

125 doc="Number of iterations of detect sources, measure sources, " 

126 "estimate PSF. If useSimplePsf is True then 2 should be plenty; " 

127 "otherwise more may be wanted.", 

128 ) 

129 background = pexConfig.ConfigurableField( 

130 target=SubtractBackgroundTask, 

131 doc="Configuration for initial background estimation", 

132 ) 

133 detection = pexConfig.ConfigurableField( 

134 target=SourceDetectionTask, 

135 doc="Detect sources" 

136 ) 

137 doDeblend = pexConfig.Field( 

138 dtype=bool, 

139 default=True, 

140 doc="Run deblender input exposure" 

141 ) 

142 deblend = pexConfig.ConfigurableField( 

143 target=SourceDeblendTask, 

144 doc="Split blended source into their components" 

145 ) 

146 measurement = pexConfig.ConfigurableField( 

147 target=SingleFrameMeasurementTask, 

148 doc="Measure sources" 

149 ) 

150 doApCorr = pexConfig.Field( 

151 dtype=bool, 

152 default=True, 

153 doc="Run subtasks to measure and apply aperture corrections" 

154 ) 

155 measureApCorr = pexConfig.ConfigurableField( 

156 target=MeasureApCorrTask, 

157 doc="Subtask to measure aperture corrections" 

158 ) 

159 applyApCorr = pexConfig.ConfigurableField( 

160 target=ApplyApCorrTask, 

161 doc="Subtask to apply aperture corrections" 

162 ) 

163 # If doApCorr is False, and the exposure does not have apcorrections already applied, the 

164 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin 

165 catalogCalculation = pexConfig.ConfigurableField( 

166 target=CatalogCalculationTask, 

167 doc="Subtask to run catalogCalculation plugins on catalog" 

168 ) 

169 doComputeSummaryStats = pexConfig.Field( 

170 dtype=bool, 

171 default=True, 

172 doc="Run subtask to measure exposure summary statistics", 

173 deprecated=("This subtask has been moved to CalibrateTask " 

174 "with DM-30701.") 

175 ) 

176 computeSummaryStats = pexConfig.ConfigurableField( 

177 target=ComputeExposureSummaryStatsTask, 

178 doc="Subtask to run computeSummaryStats on exposure", 

179 deprecated=("This subtask has been moved to CalibrateTask " 

180 "with DM-30701.") 

181 ) 

182 useSimplePsf = pexConfig.Field( 

183 dtype=bool, 

184 default=True, 

185 doc="Replace the existing PSF model with a simplified version that has the same sigma " 

186 "at the start of each PSF determination iteration? Doing so makes PSF determination " 

187 "converge more robustly and quickly.", 

188 ) 

189 installSimplePsf = pexConfig.ConfigurableField( 

190 target=InstallGaussianPsfTask, 

191 doc="Install a simple PSF model", 

192 ) 

193 refObjLoader = pexConfig.ConfigField( 

194 dtype=LoadReferenceObjectsConfig, 

195 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).", 

196 doc="reference object loader", 

197 ) 

198 ref_match = pexConfig.ConfigurableField( 

199 target=RefMatchTask, 

200 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).", 

201 doc="Task to load and match reference objects. Only used if measurePsf can use matches. " 

202 "Warning: matching will only work well if the initial WCS is accurate enough " 

203 "to give good matches (roughly: good to 3 arcsec across the CCD).", 

204 ) 

205 measurePsf = pexConfig.ConfigurableField( 

206 target=MeasurePsfTask, 

207 doc="Measure PSF", 

208 ) 

209 repair = pexConfig.ConfigurableField( 

210 target=RepairTask, 

211 doc="Remove cosmic rays", 

212 ) 

213 requireCrForPsf = pexConfig.Field( 

214 dtype=bool, 

215 default=True, 

216 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF." 

217 ) 

218 checkUnitsParseStrict = pexConfig.Field( 

219 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'", 

220 dtype=str, 

221 default="raise", 

222 ) 

223 

224 def setDefaults(self): 

225 super().setDefaults() 

226 # just detect bright stars; includeThresholdMultipler=10 seems large, 

227 # but these are the values we have been using 

228 self.detection.thresholdValue = 5.0 

229 self.detection.includeThresholdMultiplier = 10.0 

230 self.detection.doTempLocalBackground = False 

231 # do not deblend, as it makes a mess 

232 self.doDeblend = False 

233 # measure and apply aperture correction; note: measuring and applying aperture 

234 # correction are disabled until the final measurement, after PSF is measured 

235 self.doApCorr = True 

236 # During characterization, we don't have full source measurement information, 

237 # so must do the aperture correction with only psf stars, combined with the 

238 # default signal-to-noise cuts in MeasureApCorrTask. 

239 selector = self.measureApCorr.sourceSelector["science"] 

240 selector.doUnresolved = False 

241 selector.flags.good = ["calib_psf_used"] 

242 selector.flags.bad = [] 

243 

244 # minimal set of measurements needed to determine PSF 

245 self.measurement.plugins.names = [ 

246 "base_PixelFlags", 

247 "base_SdssCentroid", 

248 "ext_shapeHSM_HsmSourceMoments", 

249 "base_GaussianFlux", 

250 "base_PsfFlux", 

251 "base_CircularApertureFlux", 

252 ] 

253 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

254 

255 def validate(self): 

256 if self.doApCorr and not self.measurePsf: 

257 raise RuntimeError("Must measure PSF to measure aperture correction, " 

258 "because flags determined by PSF measurement are used to identify " 

259 "sources used to measure aperture correction") 

260 

261 

262class CharacterizeImageTask(pipeBase.PipelineTask): 

263 """Measure bright sources and use this to estimate background and PSF of 

264 an exposure. 

265 

266 Given an exposure with defects repaired (masked and interpolated over, 

267 e.g. as output by `~lsst.ip.isr.IsrTask`): 

268 - detect and measure bright sources 

269 - repair cosmic rays 

270 - measure and subtract background 

271 - measure PSF 

272 

273 Parameters 

274 ---------- 

275 butler : `None` 

276 Compatibility parameter. Should always be `None`. 

277 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

278 Reference object loader if using a catalog-based star-selector. 

279 schema : `lsst.afw.table.Schema`, optional 

280 Initial schema for icSrc catalog. 

281 **kwargs 

282 Additional keyword arguments. 

283 

284 Notes 

285 ----- 

286 Debugging: 

287 CharacterizeImageTask has a debug dictionary with the following keys: 

288 

289 frame 

290 int: if specified, the frame of first debug image displayed (defaults to 1) 

291 repair_iter 

292 bool; if True display image after each repair in the measure PSF loop 

293 background_iter 

294 bool; if True display image after each background subtraction in the measure PSF loop 

295 measure_iter 

296 bool; if True display image and sources at the end of each iteration of the measure PSF loop 

297 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols. 

298 psf 

299 bool; if True display image and sources after PSF is measured; 

300 this will be identical to the final image displayed by measure_iter if measure_iter is true 

301 repair 

302 bool; if True display image and sources after final repair 

303 measure 

304 bool; if True display image and sources after final measurement 

305 """ 

306 

307 ConfigClass = CharacterizeImageConfig 

308 _DefaultName = "characterizeImage" 

309 

310 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): 

311 super().__init__(**kwargs) 

312 

313 if butler is not None: 

314 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

315 category=FutureWarning, stacklevel=2) 

316 butler = None 

317 

318 if schema is None: 

319 schema = SourceTable.makeMinimalSchema() 

320 self.schema = schema 

321 self.makeSubtask("background") 

322 self.makeSubtask("installSimplePsf") 

323 self.makeSubtask("repair") 

324 self.makeSubtask("measurePsf", schema=self.schema) 

325 # TODO DM-34769: remove this `if` block 

326 if self.config.doMeasurePsf and self.measurePsf.usesMatches: 

327 self.makeSubtask("ref_match", refObjLoader=refObjLoader) 

328 self.algMetadata = dafBase.PropertyList() 

329 self.makeSubtask('detection', schema=self.schema) 

330 if self.config.doDeblend: 

331 self.makeSubtask("deblend", schema=self.schema) 

332 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata) 

333 if self.config.doApCorr: 

334 self.makeSubtask('measureApCorr', schema=self.schema) 

335 self.makeSubtask('applyApCorr', schema=self.schema) 

336 self.makeSubtask('catalogCalculation', schema=self.schema) 

337 self._initialFrame = getDebugFrame(self._display, "frame") or 1 

338 self._frame = self._initialFrame 

339 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

340 self.outputSchema = afwTable.SourceCatalog(self.schema) 

341 

342 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

343 inputs = butlerQC.get(inputRefs) 

344 if 'exposureIdInfo' not in inputs.keys(): 

345 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

346 outputs = self.run(**inputs) 

347 butlerQC.put(outputs, outputRefs) 

348 

349 @timeMethod 

350 def run(self, exposure, exposureIdInfo=None, background=None): 

351 """Characterize a science image. 

352 

353 Peforms the following operations: 

354 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: 

355 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) 

356 - interpolate over cosmic rays 

357 - perform final measurement 

358 

359 Parameters 

360 ---------- 

361 exposure : `lsst.afw.image.ExposureF` 

362 Exposure to characterize. 

363 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

364 Exposure ID info. If not provided, returned SourceCatalog IDs will not 

365 be globally unique. 

366 background : `lsst.afw.math.BackgroundList`, optional 

367 Initial model of background already subtracted from exposure. 

368 

369 Returns 

370 ------- 

371 result : `lsst.pipe.base.Struct` 

372 Results as a struct with attributes: 

373 

374 ``exposure`` 

375 Characterized exposure (`lsst.afw.image.ExposureF`). 

376 ``sourceCat`` 

377 Detected sources (`lsst.afw.table.SourceCatalog`). 

378 ``background`` 

379 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

380 ``psfCellSet`` 

381 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

382 ``characterized`` 

383 Another reference to ``exposure`` for compatibility. 

384 ``backgroundModel`` 

385 Another reference to ``background`` for compatibility. 

386 

387 Raises 

388 ------ 

389 RuntimeError 

390 Raised if PSF sigma is NaN. 

391 """ 

392 self._frame = self._initialFrame # reset debug display frame 

393 

394 if not self.config.doMeasurePsf and not exposure.hasPsf(): 

395 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.") 

396 self.installSimplePsf.run(exposure=exposure) 

397 

398 if exposureIdInfo is None: 

399 exposureIdInfo = ExposureIdInfo() 

400 

401 # subtract an initial estimate of background level 

402 background = self.background.run(exposure).background 

403 

404 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1 

405 for i in range(psfIterations): 

406 dmeRes = self.detectMeasureAndEstimatePsf( 

407 exposure=exposure, 

408 exposureIdInfo=exposureIdInfo, 

409 background=background, 

410 ) 

411 

412 psf = dmeRes.exposure.getPsf() 

413 # Just need a rough estimate; average positions are fine 

414 psfAvgPos = psf.getAveragePosition() 

415 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius() 

416 psfDimensions = psf.computeImage(psfAvgPos).getDimensions() 

417 medBackground = np.median(dmeRes.background.getImage().getArray()) 

418 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f", 

419 i + 1, psfSigma, psfDimensions, medBackground) 

420 if np.isnan(psfSigma): 

421 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.") 

422 

423 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

424 

425 # perform final repair with final PSF 

426 self.repair.run(exposure=dmeRes.exposure) 

427 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

428 

429 # perform final measurement with final PSF, including measuring and applying aperture correction, 

430 # if wanted 

431 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, 

432 exposureId=exposureIdInfo.expId) 

433 if self.config.doApCorr: 

434 try: 

435 apCorrMap = self.measureApCorr.run( 

436 exposure=dmeRes.exposure, 

437 catalog=dmeRes.sourceCat, 

438 ).apCorrMap 

439 except MeasureApCorrError: 

440 # We have failed to get a valid aperture correction map. 

441 # Proceed with processing, and image will be filtered 

442 # downstream. 

443 dmeRes.exposure.info.setApCorrMap(None) 

444 else: 

445 dmeRes.exposure.info.setApCorrMap(apCorrMap) 

446 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) 

447 

448 self.catalogCalculation.run(dmeRes.sourceCat) 

449 

450 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

451 

452 return pipeBase.Struct( 

453 exposure=dmeRes.exposure, 

454 sourceCat=dmeRes.sourceCat, 

455 background=dmeRes.background, 

456 psfCellSet=dmeRes.psfCellSet, 

457 

458 characterized=dmeRes.exposure, 

459 backgroundModel=dmeRes.background 

460 ) 

461 

462 @timeMethod 

463 def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background): 

464 """Perform one iteration of detect, measure, and estimate PSF. 

465 

466 Performs the following operations: 

467 

468 - if config.doMeasurePsf or not exposure.hasPsf(): 

469 

470 - install a simple PSF model (replacing the existing one, if need be) 

471 

472 - interpolate over cosmic rays with keepCRs=True 

473 - estimate background and subtract it from the exposure 

474 - detect, deblend and measure sources, and subtract a refined background model; 

475 - if config.doMeasurePsf: 

476 - measure PSF 

477 

478 Parameters 

479 ---------- 

480 exposure : `lsst.afw.image.ExposureF` 

481 Exposure to characterize. 

482 exposureIdInfo : `lsst.obs.baseExposureIdInfo` 

483 Exposure ID info. 

484 background : `lsst.afw.math.BackgroundList`, optional 

485 Initial model of background already subtracted from exposure. 

486 

487 Returns 

488 ------- 

489 result : `lsst.pipe.base.Struct` 

490 Results as a struct with attributes: 

491 

492 ``exposure`` 

493 Characterized exposure (`lsst.afw.image.ExposureF`). 

494 ``sourceCat`` 

495 Detected sources (`lsst.afw.table.SourceCatalog`). 

496 ``background`` 

497 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

498 ``psfCellSet`` 

499 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

500 

501 Raises 

502 ------ 

503 LengthError 

504 Raised if there are too many CR pixels. 

505 """ 

506 # install a simple PSF model, if needed or wanted 

507 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): 

508 self.log.info("PSF estimation initialized with 'simple' PSF") 

509 self.installSimplePsf.run(exposure=exposure) 

510 

511 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model) 

512 if self.config.requireCrForPsf: 

513 self.repair.run(exposure=exposure, keepCRs=True) 

514 else: 

515 try: 

516 self.repair.run(exposure=exposure, keepCRs=True) 

517 except LengthError: 

518 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)", 

519 self.config.repair.cosmicray.nCrPixelMax) 

520 

521 self.display("repair_iter", exposure=exposure) 

522 

523 if background is None: 

524 background = BackgroundList() 

525 

526 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

527 table = SourceTable.make(self.schema, sourceIdFactory) 

528 table.setMetadata(self.algMetadata) 

529 

530 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) 

531 sourceCat = detRes.sources 

532 if detRes.background: 

533 for bg in detRes.background: 

534 background.append(bg) 

535 

536 if self.config.doDeblend: 

537 self.deblend.run(exposure=exposure, sources=sourceCat) 

538 

539 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId) 

540 

541 measPsfRes = pipeBase.Struct(cellSet=None) 

542 if self.config.doMeasurePsf: 

543 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below. 

544 if self.measurePsf.usesMatches: 

545 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches 

546 else: 

547 matches = None 

548 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches, 

549 expId=exposureIdInfo.expId) 

550 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat) 

551 

552 return pipeBase.Struct( 

553 exposure=exposure, 

554 sourceCat=sourceCat, 

555 background=background, 

556 psfCellSet=measPsfRes.cellSet, 

557 ) 

558 

559 def display(self, itemName, exposure, sourceCat=None): 

560 """Display exposure and sources on next frame (for debugging). 

561 

562 Parameters 

563 ---------- 

564 itemName : `str` 

565 Name of item in ``debugInfo``. 

566 exposure : `lsst.afw.image.ExposureF` 

567 Exposure to display. 

568 sourceCat : `lsst.afw.table.SourceCatalog`, optional 

569 Catalog of sources detected on the exposure. 

570 """ 

571 val = getDebugFrame(self._display, itemName) 

572 if not val: 

573 return 

574 

575 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False) 

576 self._frame += 1