Coverage for python/lsst/pipe/tasks/characterizeImage.py: 32%

181 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-26 03:18 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy as np 

23import warnings 

24 

25from lsstDebug import getDebugFrame 

26import lsst.afw.table as afwTable 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29import lsst.daf.base as dafBase 

30import lsst.pipe.base.connectionTypes as cT 

31from lsst.afw.math import BackgroundList 

32from lsst.afw.table import SourceTable, SourceCatalog 

33from lsst.meas.algorithms import SubtractBackgroundTask, SourceDetectionTask, MeasureApCorrTask 

34from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask 

35from lsst.meas.astrom import RefMatchTask, displayAstrometry 

36from lsst.meas.algorithms import LoadReferenceObjectsConfig 

37from lsst.obs.base import ExposureIdInfo 

38from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask 

39from lsst.meas.deblender import SourceDeblendTask 

40import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin 

41from .measurePsf import MeasurePsfTask 

42from .repair import RepairTask 

43from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

44from lsst.pex.exceptions import LengthError 

45from lsst.utils.timer import timeMethod 

46 

47__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"] 

48 

49 

50class CharacterizeImageConnections(pipeBase.PipelineTaskConnections, 

51 dimensions=("instrument", "visit", "detector")): 

52 exposure = cT.Input( 

53 doc="Input exposure data", 

54 name="postISRCCD", 

55 storageClass="Exposure", 

56 dimensions=["instrument", "exposure", "detector"], 

57 ) 

58 characterized = cT.Output( 

59 doc="Output characterized data.", 

60 name="icExp", 

61 storageClass="ExposureF", 

62 dimensions=["instrument", "visit", "detector"], 

63 ) 

64 sourceCat = cT.Output( 

65 doc="Output source catalog.", 

66 name="icSrc", 

67 storageClass="SourceCatalog", 

68 dimensions=["instrument", "visit", "detector"], 

69 ) 

70 backgroundModel = cT.Output( 

71 doc="Output background model.", 

72 name="icExpBackground", 

73 storageClass="Background", 

74 dimensions=["instrument", "visit", "detector"], 

75 ) 

76 outputSchema = cT.InitOutput( 

77 doc="Schema of the catalog produced by CharacterizeImage", 

78 name="icSrc_schema", 

79 storageClass="SourceCatalog", 

80 ) 

81 

82 def adjustQuantum(self, inputs, outputs, label, dataId): 

83 # Docstring inherited from PipelineTaskConnections 

84 try: 

85 return super().adjustQuantum(inputs, outputs, label, dataId) 

86 except pipeBase.ScalarError as err: 

87 raise pipeBase.ScalarError( 

88 "CharacterizeImageTask can at present only be run on visits that are associated with " 

89 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the " 

90 "snap-combination step you probably want hasn't been configured to run between ISR and " 

91 "this task (as of this writing, that would be because it hasn't been implemented yet)." 

92 ) from err 

93 

94 

95class CharacterizeImageConfig(pipeBase.PipelineTaskConfig, 

96 pipelineConnections=CharacterizeImageConnections): 

97 

98 """!Config for CharacterizeImageTask""" 

99 doMeasurePsf = pexConfig.Field( 

100 dtype=bool, 

101 default=True, 

102 doc="Measure PSF? If False then for all subsequent operations use either existing PSF " 

103 "model when present, or install simple PSF model when not (see installSimplePsf " 

104 "config options)" 

105 ) 

106 doWrite = pexConfig.Field( 

107 dtype=bool, 

108 default=True, 

109 doc="Persist results?", 

110 ) 

111 doWriteExposure = pexConfig.Field( 

112 dtype=bool, 

113 default=True, 

114 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.", 

115 ) 

116 psfIterations = pexConfig.RangeField( 

117 dtype=int, 

118 default=2, 

119 min=1, 

120 doc="Number of iterations of detect sources, measure sources, " 

121 "estimate PSF. If useSimplePsf is True then 2 should be plenty; " 

122 "otherwise more may be wanted.", 

123 ) 

124 background = pexConfig.ConfigurableField( 

125 target=SubtractBackgroundTask, 

126 doc="Configuration for initial background estimation", 

127 ) 

128 detection = pexConfig.ConfigurableField( 

129 target=SourceDetectionTask, 

130 doc="Detect sources" 

131 ) 

132 doDeblend = pexConfig.Field( 

133 dtype=bool, 

134 default=True, 

135 doc="Run deblender input exposure" 

136 ) 

137 deblend = pexConfig.ConfigurableField( 

138 target=SourceDeblendTask, 

139 doc="Split blended source into their components" 

140 ) 

141 measurement = pexConfig.ConfigurableField( 

142 target=SingleFrameMeasurementTask, 

143 doc="Measure sources" 

144 ) 

145 doApCorr = pexConfig.Field( 

146 dtype=bool, 

147 default=True, 

148 doc="Run subtasks to measure and apply aperture corrections" 

149 ) 

150 measureApCorr = pexConfig.ConfigurableField( 

151 target=MeasureApCorrTask, 

152 doc="Subtask to measure aperture corrections" 

153 ) 

154 applyApCorr = pexConfig.ConfigurableField( 

155 target=ApplyApCorrTask, 

156 doc="Subtask to apply aperture corrections" 

157 ) 

158 # If doApCorr is False, and the exposure does not have apcorrections already applied, the 

159 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin 

160 catalogCalculation = pexConfig.ConfigurableField( 

161 target=CatalogCalculationTask, 

162 doc="Subtask to run catalogCalculation plugins on catalog" 

163 ) 

164 doComputeSummaryStats = pexConfig.Field( 

165 dtype=bool, 

166 default=True, 

167 doc="Run subtask to measure exposure summary statistics", 

168 deprecated=("This subtask has been moved to CalibrateTask " 

169 "with DM-30701.") 

170 ) 

171 computeSummaryStats = pexConfig.ConfigurableField( 

172 target=ComputeExposureSummaryStatsTask, 

173 doc="Subtask to run computeSummaryStats on exposure", 

174 deprecated=("This subtask has been moved to CalibrateTask " 

175 "with DM-30701.") 

176 ) 

177 useSimplePsf = pexConfig.Field( 

178 dtype=bool, 

179 default=True, 

180 doc="Replace the existing PSF model with a simplified version that has the same sigma " 

181 "at the start of each PSF determination iteration? Doing so makes PSF determination " 

182 "converge more robustly and quickly.", 

183 ) 

184 installSimplePsf = pexConfig.ConfigurableField( 

185 target=InstallGaussianPsfTask, 

186 doc="Install a simple PSF model", 

187 ) 

188 refObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).", 

191 doc="reference object loader", 

192 ) 

193 ref_match = pexConfig.ConfigurableField( 

194 target=RefMatchTask, 

195 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).", 

196 doc="Task to load and match reference objects. Only used if measurePsf can use matches. " 

197 "Warning: matching will only work well if the initial WCS is accurate enough " 

198 "to give good matches (roughly: good to 3 arcsec across the CCD).", 

199 ) 

200 measurePsf = pexConfig.ConfigurableField( 

201 target=MeasurePsfTask, 

202 doc="Measure PSF", 

203 ) 

204 repair = pexConfig.ConfigurableField( 

205 target=RepairTask, 

206 doc="Remove cosmic rays", 

207 ) 

208 requireCrForPsf = pexConfig.Field( 

209 dtype=bool, 

210 default=True, 

211 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF." 

212 ) 

213 checkUnitsParseStrict = pexConfig.Field( 

214 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'", 

215 dtype=str, 

216 default="raise", 

217 ) 

218 

219 def setDefaults(self): 

220 super().setDefaults() 

221 # just detect bright stars; includeThresholdMultipler=10 seems large, 

222 # but these are the values we have been using 

223 self.detection.thresholdValue = 5.0 

224 self.detection.includeThresholdMultiplier = 10.0 

225 self.detection.doTempLocalBackground = False 

226 # do not deblend, as it makes a mess 

227 self.doDeblend = False 

228 # measure and apply aperture correction; note: measuring and applying aperture 

229 # correction are disabled until the final measurement, after PSF is measured 

230 self.doApCorr = True 

231 # minimal set of measurements needed to determine PSF 

232 self.measurement.plugins.names = [ 

233 "base_PixelFlags", 

234 "base_SdssCentroid", 

235 "ext_shapeHSM_HsmSourceMoments", 

236 "base_GaussianFlux", 

237 "base_PsfFlux", 

238 "base_CircularApertureFlux", 

239 ] 

240 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" 

241 

242 def validate(self): 

243 if self.doApCorr and not self.measurePsf: 

244 raise RuntimeError("Must measure PSF to measure aperture correction, " 

245 "because flags determined by PSF measurement are used to identify " 

246 "sources used to measure aperture correction") 

247 

248 

249class CharacterizeImageTask(pipeBase.PipelineTask): 

250 """Measure bright sources and use this to estimate background and PSF of an exposure. 

251 

252 Parameters 

253 ---------- 

254 butler : `None` 

255 Compatibility parameter. Should always be `None`. 

256 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

257 Reference object loader if using a catalog-based star-selector. 

258 schema : `lsst.afw.table.Schema`, optional 

259 Initial schema for icSrc catalog. 

260 """ 

261 

262 ConfigClass = CharacterizeImageConfig 

263 _DefaultName = "characterizeImage" 

264 

265 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): 

266 super().__init__(**kwargs) 

267 

268 if butler is not None: 

269 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

270 category=FutureWarning, stacklevel=2) 

271 butler = None 

272 

273 if schema is None: 

274 schema = SourceTable.makeMinimalSchema() 

275 self.schema = schema 

276 self.makeSubtask("background") 

277 self.makeSubtask("installSimplePsf") 

278 self.makeSubtask("repair") 

279 self.makeSubtask("measurePsf", schema=self.schema) 

280 # TODO DM-34769: remove this `if` block 

281 if self.config.doMeasurePsf and self.measurePsf.usesMatches: 

282 self.makeSubtask("ref_match", refObjLoader=refObjLoader) 

283 self.algMetadata = dafBase.PropertyList() 

284 self.makeSubtask('detection', schema=self.schema) 

285 if self.config.doDeblend: 

286 self.makeSubtask("deblend", schema=self.schema) 

287 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata) 

288 if self.config.doApCorr: 

289 self.makeSubtask('measureApCorr', schema=self.schema) 

290 self.makeSubtask('applyApCorr', schema=self.schema) 

291 self.makeSubtask('catalogCalculation', schema=self.schema) 

292 self._initialFrame = getDebugFrame(self._display, "frame") or 1 

293 self._frame = self._initialFrame 

294 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

295 self.outputSchema = afwTable.SourceCatalog(self.schema) 

296 

297 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

298 inputs = butlerQC.get(inputRefs) 

299 if 'exposureIdInfo' not in inputs.keys(): 

300 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

301 outputs = self.run(**inputs) 

302 butlerQC.put(outputs, outputRefs) 

303 

304 @timeMethod 

305 def run(self, exposure, exposureIdInfo=None, background=None): 

306 """Characterize a science image. 

307 

308 Peforms the following operations: 

309 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false: 

310 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details) 

311 - interpolate over cosmic rays 

312 - perform final measurement 

313 

314 Parameters 

315 ---------- 

316 exposure : `lsst.afw.image.ExposureF` 

317 Exposure to characterize. 

318 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

319 Exposure ID info. If not provided, returned SourceCatalog IDs will not 

320 be globally unique. 

321 background : `lsst.afw.math.BackgroundList`, optional 

322 Initial model of background already subtracted from exposure. 

323 

324 Returns 

325 ------- 

326 result : `lsst.pipe.base.Struct` 

327 Result structure with the following attributes: 

328 

329 ``exposure`` 

330 Characterized exposure (`lsst.afw.image.ExposureF`). 

331 ``sourceCat`` 

332 Detected sources (`lsst.afw.table.SourceCatalog`). 

333 ``background`` 

334 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

335 ``psfCellSet`` 

336 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

337 ``characterized`` 

338 Another reference to ``exposure`` for compatibility. 

339 ``backgroundModel`` 

340 Another reference to ``background`` for compatibility. 

341 """ 

342 self._frame = self._initialFrame # reset debug display frame 

343 

344 if not self.config.doMeasurePsf and not exposure.hasPsf(): 

345 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.") 

346 self.installSimplePsf.run(exposure=exposure) 

347 

348 if exposureIdInfo is None: 

349 exposureIdInfo = ExposureIdInfo() 

350 

351 # subtract an initial estimate of background level 

352 background = self.background.run(exposure).background 

353 

354 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1 

355 for i in range(psfIterations): 

356 dmeRes = self.detectMeasureAndEstimatePsf( 

357 exposure=exposure, 

358 exposureIdInfo=exposureIdInfo, 

359 background=background, 

360 ) 

361 

362 psf = dmeRes.exposure.getPsf() 

363 # Just need a rough estimate; average positions are fine 

364 psfAvgPos = psf.getAveragePosition() 

365 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius() 

366 psfDimensions = psf.computeImage(psfAvgPos).getDimensions() 

367 medBackground = np.median(dmeRes.background.getImage().getArray()) 

368 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f", 

369 i + 1, psfSigma, psfDimensions, medBackground) 

370 if np.isnan(psfSigma): 

371 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.") 

372 

373 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

374 

375 # perform final repair with final PSF 

376 self.repair.run(exposure=dmeRes.exposure) 

377 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

378 

379 # perform final measurement with final PSF, including measuring and applying aperture correction, 

380 # if wanted 

381 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, 

382 exposureId=exposureIdInfo.expId) 

383 if self.config.doApCorr: 

384 apCorrMap = self.measureApCorr.run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap 

385 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap) 

386 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap()) 

387 self.catalogCalculation.run(dmeRes.sourceCat) 

388 

389 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) 

390 

391 return pipeBase.Struct( 

392 exposure=dmeRes.exposure, 

393 sourceCat=dmeRes.sourceCat, 

394 background=dmeRes.background, 

395 psfCellSet=dmeRes.psfCellSet, 

396 

397 characterized=dmeRes.exposure, 

398 backgroundModel=dmeRes.background 

399 ) 

400 

401 @timeMethod 

402 def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background): 

403 """Perform one iteration of detect, measure, and estimate PSF. 

404 

405 Performs the following operations: 

406 - if config.doMeasurePsf or not exposure.hasPsf(): 

407 - install a simple PSF model (replacing the existing one, if need be) 

408 - interpolate over cosmic rays with keepCRs=True 

409 - estimate background and subtract it from the exposure 

410 - detect, deblend and measure sources, and subtract a refined background model; 

411 - if config.doMeasurePsf: 

412 - measure PSF 

413 

414 Parameters 

415 ---------- 

416 exposure : `lsst.afw.image.ExposureF` 

417 Exposure to characterize. 

418 exposureIdInfo : `lsst.obs.baseExposureIdInfo` 

419 Exposure ID info. 

420 background : `lsst.afw.math.BackgroundList`, optional 

421 Initial model of background already subtracted from exposure. 

422 

423 Returns 

424 ------- 

425 result : `lsst.pipe.base.Struct` 

426 Result structure with the following attributes: 

427 

428 ``exposure`` 

429 Characterized exposure (`lsst.afw.image.ExposureF`). 

430 ``sourceCat`` 

431 Detected sources (`lsst.afw.table.SourceCatalog`). 

432 ``background`` 

433 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

434 ``psfCellSet`` 

435 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`). 

436 """ 

437 # install a simple PSF model, if needed or wanted 

438 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf): 

439 self.log.info("PSF estimation initialized with 'simple' PSF") 

440 self.installSimplePsf.run(exposure=exposure) 

441 

442 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model) 

443 if self.config.requireCrForPsf: 

444 self.repair.run(exposure=exposure, keepCRs=True) 

445 else: 

446 try: 

447 self.repair.run(exposure=exposure, keepCRs=True) 

448 except LengthError: 

449 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)", 

450 self.config.repair.cosmicray.nCrPixelMax) 

451 

452 self.display("repair_iter", exposure=exposure) 

453 

454 if background is None: 

455 background = BackgroundList() 

456 

457 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

458 table = SourceTable.make(self.schema, sourceIdFactory) 

459 table.setMetadata(self.algMetadata) 

460 

461 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True) 

462 sourceCat = detRes.sources 

463 if detRes.fpSets.background: 

464 for bg in detRes.fpSets.background: 

465 background.append(bg) 

466 

467 if self.config.doDeblend: 

468 self.deblend.run(exposure=exposure, sources=sourceCat) 

469 

470 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId) 

471 

472 measPsfRes = pipeBase.Struct(cellSet=None) 

473 if self.config.doMeasurePsf: 

474 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below. 

475 if self.measurePsf.usesMatches: 

476 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches 

477 else: 

478 matches = None 

479 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches, 

480 expId=exposureIdInfo.expId) 

481 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat) 

482 

483 return pipeBase.Struct( 

484 exposure=exposure, 

485 sourceCat=sourceCat, 

486 background=background, 

487 psfCellSet=measPsfRes.cellSet, 

488 ) 

489 

490 def getSchemaCatalogs(self): 

491 """Return a dict of empty catalogs for each catalog dataset produced by this task. 

492 """ 

493 sourceCat = SourceCatalog(self.schema) 

494 sourceCat.getTable().setMetadata(self.algMetadata) 

495 return {"icSrc": sourceCat} 

496 

497 def display(self, itemName, exposure, sourceCat=None): 

498 """Display exposure and sources on next frame (for debugging). 

499 

500 Parameters 

501 ---------- 

502 itemName : `str` 

503 Name of item in ``debugInfo``. 

504 exposure : `lsst.afw.image.ExposureF` 

505 Exposure to display. 

506 sourceCat : `lsst.afw.table.SourceCatalog` 

507 Exposure to display. 

508 """ 

509 val = getDebugFrame(self._display, itemName) 

510 if not val: 

511 return 

512 

513 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False) 

514 self._frame += 1