Coverage for python/lsst/pipe/tasks/characterizeImage.py: 31%

177 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-08 03:36 -0800

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"] 

23 

24import numpy as np 

25import warnings 

26 

27from lsstDebug import getDebugFrame 

28import lsst.afw.table as afwTable 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.daf.base as dafBase 

32import lsst.pipe.base.connectionTypes as cT 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SubtractBackgroundTask, SourceDetectionTask, MeasureApCorrTask 

36from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask 

37from lsst.meas.astrom import RefMatchTask, displayAstrometry 

38from lsst.meas.algorithms import LoadReferenceObjectsConfig 

39from lsst.obs.base import ExposureIdInfo 

40from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask 

41from lsst.meas.deblender import SourceDeblendTask 

42import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin 

43from .measurePsf import MeasurePsfTask 

44from .repair import RepairTask 

45from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

46from lsst.pex.exceptions import LengthError 

47from lsst.utils.timer import timeMethod 

48 

49 

50class CharacterizeImageConnections(pipeBase.PipelineTaskConnections, 

51 dimensions=("instrument", "visit", "detector")): 

52 exposure = cT.Input( 

53 doc="Input exposure data", 

54 name="postISRCCD", 

55 storageClass="Exposure", 

56 dimensions=["instrument", "exposure", "detector"], 

57 ) 

58 characterized = cT.Output( 

59 doc="Output characterized data.", 

60 name="icExp", 

61 storageClass="ExposureF", 

62 dimensions=["instrument", "visit", "detector"], 

63 ) 

64 sourceCat = cT.Output( 

65 doc="Output source catalog.", 

66 name="icSrc", 

67 storageClass="SourceCatalog", 

68 dimensions=["instrument", "visit", "detector"], 

69 ) 

70 backgroundModel = cT.Output( 

71 doc="Output background model.", 

72 name="icExpBackground", 

73 storageClass="Background", 

74 dimensions=["instrument", "visit", "detector"], 

75 ) 

76 outputSchema = cT.InitOutput( 

77 doc="Schema of the catalog produced by CharacterizeImage", 

78 name="icSrc_schema", 

79 storageClass="SourceCatalog", 

80 ) 

81 

82 def adjustQuantum(self, inputs, outputs, label, dataId): 

83 # Docstring inherited from PipelineTaskConnections 

84 try: 

85 return super().adjustQuantum(inputs, outputs, label, dataId) 

86 except pipeBase.ScalarError as err: 

87 raise pipeBase.ScalarError( 

88 "CharacterizeImageTask can at present only be run on visits that are associated with " 

89 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the " 

90 "snap-combination step you probably want hasn't been configured to run between ISR and " 

91 "this task (as of this writing, that would be because it hasn't been implemented yet)." 

92 ) from err 

93 

94 

95class CharacterizeImageConfig(pipeBase.PipelineTaskConfig, 

96 pipelineConnections=CharacterizeImageConnections): 

97 """Config for CharacterizeImageTask.""" 

98 

99 doMeasurePsf = pexConfig.Field( 

100 dtype=bool, 

101 default=True, 

102 doc="Measure PSF? If False then for all subsequent operations use either existing PSF " 

103 "model when present, or install simple PSF model when not (see installSimplePsf " 

104 "config options)" 

105 ) 

106 doWrite = pexConfig.Field( 

107 dtype=bool, 

108 default=True, 

109 doc="Persist results?", 

110 ) 

111 doWriteExposure = pexConfig.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.", 

115 ) 

116 psfIterations = pexConfig.RangeField( 

117 dtype=int, 

118 default=2, 

119 min=1, 

120 doc="Number of iterations of detect sources, measure sources, " 

121 "estimate PSF. If useSimplePsf is True then 2 should be plenty; " 

122 "otherwise more may be wanted.", 

123 ) 

124 background = pexConfig.ConfigurableField( 

125 target=SubtractBackgroundTask, 

126 doc="Configuration for initial background estimation", 

127 ) 

128 detection = pexConfig.ConfigurableField( 

129 target=SourceDetectionTask, 

130 doc="Detect sources" 

131 ) 

132 doDeblend = pexConfig.Field( 

133 dtype=bool, 

134 default=True, 

135 doc="Run deblender input exposure" 

136 ) 

137 deblend = pexConfig.ConfigurableField( 

138 target=SourceDeblendTask, 

139 doc="Split blended source into their components" 

140 ) 

141 measurement = pexConfig.ConfigurableField( 

142 target=SingleFrameMeasurementTask, 

143 doc="Measure sources" 

144 ) 

145 doApCorr = pexConfig.Field( 

146 dtype=bool, 

147 default=True, 

148 doc="Run subtasks to measure and apply aperture corrections" 

149 ) 

150 measureApCorr = pexConfig.ConfigurableField( 

151 target=MeasureApCorrTask, 

152 doc="Subtask to measure aperture corrections" 

153 ) 

154 applyApCorr = pexConfig.ConfigurableField( 

155 target=ApplyApCorrTask, 

156 doc="Subtask to apply aperture corrections" 

157 ) 

158 # If doApCorr is False, and the exposure does not have apcorrections already applied, the 

159 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin 

160 catalogCalculation = pexConfig.ConfigurableField( 

161 target=CatalogCalculationTask, 

162 doc="Subtask to run catalogCalculation plugins on catalog" 

163 ) 

164 doComputeSummaryStats = pexConfig.Field( 

165 dtype=bool, 

166 default=True, 

167 doc="Run subtask to measure exposure summary statistics", 

168 deprecated=("This subtask has been moved to CalibrateTask " 

169 "with DM-30701.") 

170 ) 

171 computeSummaryStats = pexConfig.ConfigurableField( 

172 target=ComputeExposureSummaryStatsTask, 

173 doc="Subtask to run computeSummaryStats on exposure", 

174 deprecated=("This subtask has been moved to CalibrateTask " 

175 "with DM-30701.") 

176 ) 

177 useSimplePsf = pexConfig.Field( 

178 dtype=bool, 

179 default=True, 

180 doc="Replace the existing PSF model with a simplified version that has the same sigma " 

181 "at the start of each PSF determination iteration? Doing so makes PSF determination " 

182 "converge more robustly and quickly.", 

183 ) 

184 installSimplePsf = pexConfig.ConfigurableField( 

185 target=InstallGaussianPsfTask, 

186 doc="Install a simple PSF model", 

187 ) 

188 refObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).", 

191 doc="reference object loader", 

192 ) 

193 ref_match = pexConfig.ConfigurableField( 

194 target=RefMatchTask, 

195 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).", 

196 doc="Task to load and match reference objects. Only used if measurePsf can use matches. " 

197 "Warning: matching will only work well if the initial WCS is accurate enough " 

198 "to give good matches (roughly: good to 3 arcsec across the CCD).", 

199 ) 

200 measurePsf = pexConfig.ConfigurableField( 

201 target=MeasurePsfTask, 

202 doc="Measure PSF", 

203 ) 

204 repair = pexConfig.ConfigurableField( 

205 target=RepairTask, 

206 doc="Remove cosmic rays", 

207 ) 

208 requireCrForPsf = pexConfig.Field( 

209 dtype=bool, 

210 default=True, 

211 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF." 

212 ) 

213 checkUnitsParseStrict = pexConfig.Field( 

214 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'", 

215 dtype=str, 

216 default="raise", 

217 ) 

218 

219 def setDefaults(self): 

220 super().setDefaults() 

221 # just detect bright stars; includeThresholdMultipler=10 seems large, 

222 # but these are the values we have been using 

223 self.detection.thresholdValue = 5.0 

224 self.detection.includeThresholdMultiplier = 10.0 

225 self.detection.doTempLocalBackground = False 

226 # do not deblend, as it makes a mess 

227 self.doDeblend = False 

228 # measure and apply aperture correction; note: measuring and applying aperture 

229 # correction are disabled until the final measurement, after PSF is measured 

230 self.doApCorr = True 

231 # minimal set of measurements needed to determine PSF 

232 self.measurement.plugins.names = [ 

233 "base_PixelFlags", 

234 "base_SdssCentroid", 

235 "ext_shapeHSM_HsmSourceMoments", 

236 "base_GaussianFlux", 

237 "base_PsfFlux", 

238 "base_CircularApertureFlux", 

239 ] 

240 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

241 

242 def validate(self): 

243 if self.doApCorr and not self.measurePsf: 

244 raise RuntimeError("Must measure PSF to measure aperture correction, " 

245 "because flags determined by PSF measurement are used to identify " 

246 "sources used to measure aperture correction") 

247 

248 

249class CharacterizeImageTask(pipeBase.PipelineTask): 

250 """Measure bright sources and use this to estimate background and PSF of 

251 an exposure. 

252 

253 Given an exposure with defects repaired (masked and interpolated over, 

254 e.g. as output by `~lsst.ip.isr.IsrTask`): 

255 - detect and measure bright sources 

256 - repair cosmic rays 

257 - measure and subtract background 

258 - measure PSF 

259 

260 Parameters 

261 ---------- 

262 butler : `None` 

263 Compatibility parameter. Should always be `None`. 

264 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

265 Reference object loader if using a catalog-based star-selector. 

266 schema : `lsst.afw.table.Schema`, optional 

267 Initial schema for icSrc catalog. 

268 **kwargs 

269 Additional keyword arguments. 

270 

271 Notes 

272 ----- 

273 Debugging: 

274 CharacterizeImageTask has a debug dictionary with the following keys: 

275 

276 frame 

277 int: if specified, the frame of first debug image displayed (defaults to 1) 

278 repair_iter 

279 bool; if True display image after each repair in the measure PSF loop 

280 background_iter 

281 bool; if True display image after each background subtraction in the measure PSF loop 

282 measure_iter 

283 bool; if True display image and sources at the end of each iteration of the measure PSF loop 

284 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols. 

285 psf 

286 bool; if True display image and sources after PSF is measured; 

287 this will be identical to the final image displayed by measure_iter if measure_iter is true 

288 repair 

289 bool; if True display image and sources after final repair 

290 measure 

291 bool; if True display image and sources after final measurement 

292 """ 

293 

294 ConfigClass = CharacterizeImageConfig 

295 _DefaultName = "characterizeImage" 

296 

297 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): 

298 super().__init__(**kwargs) 

299 

300 if butler is not None: 

301 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

302 category=FutureWarning, stacklevel=2) 

303 butler = None 

304 

305 if schema is None: 

306 schema = SourceTable.makeMinimalSchema() 

307 self.schema = schema 

308 self.makeSubtask("background") 

309 self.makeSubtask("installSimplePsf") 

310 self.makeSubtask("repair") 

311 self.makeSubtask("measurePsf", schema=self.schema) 

312 # TODO DM-34769: remove this `if` block 

313 if self.config.doMeasurePsf and self.measurePsf.usesMatches: 

314 self.makeSubtask("ref_match", refObjLoader=refObjLoader) 

315 self.algMetadata = dafBase.PropertyList() 

316 self.makeSubtask('detection', schema=self.schema) 

317 if self.config.doDeblend: 

318 self.makeSubtask("deblend", schema=self.schema) 

319 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata) 

320 if self.config.doApCorr: 

321 self.makeSubtask('measureApCorr', schema=self.schema) 

322 self.makeSubtask('applyApCorr', schema=self.schema) 

323 self.makeSubtask('catalogCalculation', schema=self.schema) 

324 self._initialFrame = getDebugFrame(self._display, "frame") or 1 

325 self._frame = self._initialFrame 

326 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

327 self.outputSchema = afwTable.SourceCatalog(self.schema) 

328 

329 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

330 inputs = butlerQC.get(inputRefs) 

331 if 'exposureIdInfo' not in inputs.keys(): 

332 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

333 outputs = self.run(**inputs) 

334 butlerQC.put(outputs, outputRefs) 

335 

336 @timeMethod 

337 def run(self, exposure, exposureIdInfo=None, background=None): 

338 """Characterize a science image. 

339 

340 Peforms the following operations: 

341 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: 

342 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) 

343 - interpolate over cosmic rays 

344 - perform final measurement 

345 

346 Parameters 

347 ---------- 

348 exposure : `lsst.afw.image.ExposureF` 

349 Exposure to characterize. 

350 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

351 Exposure ID info. If not provided, returned SourceCatalog IDs will not 

352 be globally unique. 

353 background : `lsst.afw.math.BackgroundList`, optional 

354 Initial model of background already subtracted from exposure. 

355 

356 Returns 

357 ------- 

358 result : `lsst.pipe.base.Struct` 

359 Results as a struct with attributes: 

360 

361 ``exposure`` 

362 Characterized exposure (`lsst.afw.image.ExposureF`). 

363 ``sourceCat`` 

364 Detected sources (`lsst.afw.table.SourceCatalog`). 

365 ``background`` 

366 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

367 ``psfCellSet`` 

368 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

369 ``characterized`` 

370 Another reference to ``exposure`` for compatibility. 

371 ``backgroundModel`` 

372 Another reference to ``background`` for compatibility. 

373 

374 Raises 

375 ------ 

376 RuntimeError 

377 Raised if PSF sigma is NaN. 

378 """ 

379 self._frame = self._initialFrame # reset debug display frame 

380 

381 if not self.config.doMeasurePsf and not exposure.hasPsf(): 

382 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.") 

383 self.installSimplePsf.run(exposure=exposure) 

384 

385 if exposureIdInfo is None: 

386 exposureIdInfo = ExposureIdInfo() 

387 

388 # subtract an initial estimate of background level 

389 background = self.background.run(exposure).background 

390 

391 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1 

392 for i in range(psfIterations): 

393 dmeRes = self.detectMeasureAndEstimatePsf( 

394 exposure=exposure, 

395 exposureIdInfo=exposureIdInfo, 

396 background=background, 

397 ) 

398 

399 psf = dmeRes.exposure.getPsf() 

400 # Just need a rough estimate; average positions are fine 

401 psfAvgPos = psf.getAveragePosition() 

402 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius() 

403 psfDimensions = psf.computeImage(psfAvgPos).getDimensions() 

404 medBackground = np.median(dmeRes.background.getImage().getArray()) 

405 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f", 

406 i + 1, psfSigma, psfDimensions, medBackground) 

407 if np.isnan(psfSigma): 

408 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.") 

409 

410 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

411 

412 # perform final repair with final PSF 

413 self.repair.run(exposure=dmeRes.exposure) 

414 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

415 

416 # perform final measurement with final PSF, including measuring and applying aperture correction, 

417 # if wanted 

418 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, 

419 exposureId=exposureIdInfo.expId) 

420 if self.config.doApCorr: 

421 apCorrMap = self.measureApCorr.run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap 

422 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap) 

423 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) 

424 self.catalogCalculation.run(dmeRes.sourceCat) 

425 

426 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

427 

428 return pipeBase.Struct( 

429 exposure=dmeRes.exposure, 

430 sourceCat=dmeRes.sourceCat, 

431 background=dmeRes.background, 

432 psfCellSet=dmeRes.psfCellSet, 

433 

434 characterized=dmeRes.exposure, 

435 backgroundModel=dmeRes.background 

436 ) 

437 

438 @timeMethod 

439 def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background): 

440 """Perform one iteration of detect, measure, and estimate PSF. 

441 

442 Performs the following operations: 

443 

444 - if config.doMeasurePsf or not exposure.hasPsf(): 

445 

446 - install a simple PSF model (replacing the existing one, if need be) 

447 

448 - interpolate over cosmic rays with keepCRs=True 

449 - estimate background and subtract it from the exposure 

450 - detect, deblend and measure sources, and subtract a refined background model; 

451 - if config.doMeasurePsf: 

452 - measure PSF 

453 

454 Parameters 

455 ---------- 

456 exposure : `lsst.afw.image.ExposureF` 

457 Exposure to characterize. 

458 exposureIdInfo : `lsst.obs.baseExposureIdInfo` 

459 Exposure ID info. 

460 background : `lsst.afw.math.BackgroundList`, optional 

461 Initial model of background already subtracted from exposure. 

462 

463 Returns 

464 ------- 

465 result : `lsst.pipe.base.Struct` 

466 Results as a struct with attributes: 

467 

468 ``exposure`` 

469 Characterized exposure (`lsst.afw.image.ExposureF`). 

470 ``sourceCat`` 

471 Detected sources (`lsst.afw.table.SourceCatalog`). 

472 ``background`` 

473 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

474 ``psfCellSet`` 

475 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

476 

477 Raises 

478 ------ 

479 LengthError 

480 Raised if there are too many CR pixels. 

481 """ 

482 # install a simple PSF model, if needed or wanted 

483 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): 

484 self.log.info("PSF estimation initialized with 'simple' PSF") 

485 self.installSimplePsf.run(exposure=exposure) 

486 

487 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model) 

488 if self.config.requireCrForPsf: 

489 self.repair.run(exposure=exposure, keepCRs=True) 

490 else: 

491 try: 

492 self.repair.run(exposure=exposure, keepCRs=True) 

493 except LengthError: 

494 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)", 

495 self.config.repair.cosmicray.nCrPixelMax) 

496 

497 self.display("repair_iter", exposure=exposure) 

498 

499 if background is None: 

500 background = BackgroundList() 

501 

502 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

503 table = SourceTable.make(self.schema, sourceIdFactory) 

504 table.setMetadata(self.algMetadata) 

505 

506 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) 

507 sourceCat = detRes.sources 

508 if detRes.fpSets.background: 

509 for bg in detRes.fpSets.background: 

510 background.append(bg) 

511 

512 if self.config.doDeblend: 

513 self.deblend.run(exposure=exposure, sources=sourceCat) 

514 

515 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId) 

516 

517 measPsfRes = pipeBase.Struct(cellSet=None) 

518 if self.config.doMeasurePsf: 

519 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below. 

520 if self.measurePsf.usesMatches: 

521 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches 

522 else: 

523 matches = None 

524 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches, 

525 expId=exposureIdInfo.expId) 

526 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat) 

527 

528 return pipeBase.Struct( 

529 exposure=exposure, 

530 sourceCat=sourceCat, 

531 background=background, 

532 psfCellSet=measPsfRes.cellSet, 

533 ) 

534 

535 def display(self, itemName, exposure, sourceCat=None): 

536 """Display exposure and sources on next frame (for debugging). 

537 

538 Parameters 

539 ---------- 

540 itemName : `str` 

541 Name of item in ``debugInfo``. 

542 exposure : `lsst.afw.image.ExposureF` 

543 Exposure to display. 

544 sourceCat : `lsst.afw.table.SourceCatalog`, optional 

545 Catalog of sources detected on the exposure. 

546 """ 

547 val = getDebugFrame(self._display, itemName) 

548 if not val: 

549 return 

550 

551 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False) 

552 self._frame += 1