Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

320 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2024-02-08 07:10 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from lsst.utils.timer import timeMethod 

41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

42from .fakes import BaseFakeSourcesTask 

43from .photoCal import PhotoCalTask 

44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

45 

46 

47__all__ = ["CalibrateConfig", "CalibrateTask"] 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="gaia_dr2_20200414", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="ps1_pv3_3pi_20170110", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask""" 

155 doWrite = pexConfig.Field( 

156 dtype=bool, 

157 default=True, 

158 doc="Save calibration results?", 

159 ) 

160 doWriteHeavyFootprintsInSources = pexConfig.Field( 

161 dtype=bool, 

162 default=True, 

163 doc="Include HeavyFootprint data in source table? If false then heavy " 

164 "footprints are saved as normal footprints, which saves some space" 

165 ) 

166 doWriteMatches = pexConfig.Field( 

167 dtype=bool, 

168 default=True, 

169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

170 ) 

171 doWriteMatchesDenormalized = pexConfig.Field( 

172 dtype=bool, 

173 default=False, 

174 doc=("Write reference matches in denormalized format? " 

175 "This format uses more disk space, but is more convenient to " 

176 "read. Ignored if doWriteMatches=False or doWrite=False."), 

177 ) 

178 doAstrometry = pexConfig.Field( 

179 dtype=bool, 

180 default=True, 

181 doc="Perform astrometric calibration?", 

182 ) 

183 astromRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for astrometric calibration", 

186 ) 

187 photoRefObjLoader = pexConfig.ConfigurableField( 

188 target=LoadIndexedReferenceObjectsTask, 

189 doc="reference object loader for photometric calibration", 

190 ) 

191 astrometry = pexConfig.ConfigurableField( 

192 target=AstrometryTask, 

193 doc="Perform astrometric calibration to refine the WCS", 

194 ) 

195 requireAstrometry = pexConfig.Field( 

196 dtype=bool, 

197 default=True, 

198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

199 "false."), 

200 ) 

201 doPhotoCal = pexConfig.Field( 

202 dtype=bool, 

203 default=True, 

204 doc="Perform phometric calibration?", 

205 ) 

206 requirePhotoCal = pexConfig.Field( 

207 dtype=bool, 

208 default=True, 

209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

210 "false."), 

211 ) 

212 photoCal = pexConfig.ConfigurableField( 

213 target=PhotoCalTask, 

214 doc="Perform photometric calibration", 

215 ) 

216 icSourceFieldsToCopy = pexConfig.ListField( 

217 dtype=str, 

218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

219 doc=("Fields to copy from the icSource catalog to the output catalog " 

220 "for matching sources Any missing fields will trigger a " 

221 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

222 ) 

223 matchRadiusPix = pexConfig.Field( 

224 dtype=float, 

225 default=3, 

226 doc=("Match radius for matching icSourceCat objects to sourceCat " 

227 "objects (pixels)"), 

228 ) 

229 checkUnitsParseStrict = pexConfig.Field( 

230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

231 "'warn' or 'silent'"), 

232 dtype=str, 

233 default="raise", 

234 ) 

235 detection = pexConfig.ConfigurableField( 

236 target=SourceDetectionTask, 

237 doc="Detect sources" 

238 ) 

239 doDeblend = pexConfig.Field( 

240 dtype=bool, 

241 default=True, 

242 doc="Run deblender input exposure" 

243 ) 

244 deblend = pexConfig.ConfigurableField( 

245 target=SourceDeblendTask, 

246 doc="Split blended sources into their components" 

247 ) 

248 doSkySources = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Generate sky sources?", 

252 ) 

253 skySources = pexConfig.ConfigurableField( 

254 target=SkyObjectsTask, 

255 doc="Generate sky sources", 

256 ) 

257 measurement = pexConfig.ConfigurableField( 

258 target=SingleFrameMeasurementTask, 

259 doc="Measure sources" 

260 ) 

261 postCalibrationMeasurement = pexConfig.ConfigurableField( 

262 target=SingleFrameMeasurementTask, 

263 doc="Second round of measurement for plugins that need to be run after photocal" 

264 ) 

265 setPrimaryFlags = pexConfig.ConfigurableField( 

266 target=SetPrimaryFlagsTask, 

267 doc=("Set flags for primary source classification in single frame " 

268 "processing. True if sources are not sky sources and not a parent.") 

269 ) 

270 doApCorr = pexConfig.Field( 

271 dtype=bool, 

272 default=True, 

273 doc="Run subtask to apply aperture correction" 

274 ) 

275 applyApCorr = pexConfig.ConfigurableField( 

276 target=ApplyApCorrTask, 

277 doc="Subtask to apply aperture corrections" 

278 ) 

279 # If doApCorr is False, and the exposure does not have apcorrections 

280 # already applied, the active plugins in catalogCalculation almost 

281 # certainly should not contain the characterization plugin 

282 catalogCalculation = pexConfig.ConfigurableField( 

283 target=CatalogCalculationTask, 

284 doc="Subtask to run catalogCalculation plugins on catalog" 

285 ) 

286 doInsertFakes = pexConfig.Field( 

287 dtype=bool, 

288 default=False, 

289 doc="Run fake sources injection task", 

290 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. " 

291 "Please use ProcessCcdWithFakesTask instead.") 

292 ) 

293 insertFakes = pexConfig.ConfigurableField( 

294 target=BaseFakeSourcesTask, 

295 doc="Injection of fake sources for testing purposes (must be " 

296 "retargeted)", 

297 deprecated=("insertFakes is no longer supported. This config will be removed after v24. " 

298 "Please use ProcessCcdWithFakesTask instead.") 

299 ) 

300 doComputeSummaryStats = pexConfig.Field( 

301 dtype=bool, 

302 default=True, 

303 doc="Run subtask to measure exposure summary statistics?" 

304 ) 

305 computeSummaryStats = pexConfig.ConfigurableField( 

306 target=ComputeExposureSummaryStatsTask, 

307 doc="Subtask to run computeSummaryStats on exposure" 

308 ) 

309 doWriteExposure = pexConfig.Field( 

310 dtype=bool, 

311 default=True, 

312 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

313 "normal calexp but as a fakes_calexp." 

314 ) 

315 

316 def setDefaults(self): 

317 super().setDefaults() 

318 self.detection.doTempLocalBackground = False 

319 self.deblend.maxFootprintSize = 2000 

320 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

321 self.postCalibrationMeasurement.doReplaceWithNoise = False 

322 for key in self.postCalibrationMeasurement.slots: 

323 setattr(self.postCalibrationMeasurement.slots, key, None) 

324 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

325 # The photoRefCat connection is the name to use for the colorterms. 

326 self.photoCal.photoCatName = self.connections.photoRefCat 

327 

328 # NOTE: these two lines are for gen2, and are only here for compatibility. 

329 self.astromRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

330 self.photoRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

331 

332 def validate(self): 

333 super().validate() 

334 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

335 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

336 raise ValueError( 

337 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

338 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

339 ) 

340 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

341 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

342 raise ValueError( 

343 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

344 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

345 ) 

346 

347 

348## \addtogroup LSST_task_documentation 

349## \{ 

350## \page page_CalibrateTask CalibrateTask 

351## \ref CalibrateTask_ "CalibrateTask" 

352## \copybrief CalibrateTask 

353## \} 

354 

355class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

356 r"""!Calibrate an exposure: measure sources and perform astrometric and 

357 photometric calibration 

358 

359 @anchor CalibrateTask_ 

360 

361 @section pipe_tasks_calibrate_Contents Contents 

362 

363 - @ref pipe_tasks_calibrate_Purpose 

364 - @ref pipe_tasks_calibrate_Initialize 

365 - @ref pipe_tasks_calibrate_IO 

366 - @ref pipe_tasks_calibrate_Config 

367 - @ref pipe_tasks_calibrate_Metadata 

368 - @ref pipe_tasks_calibrate_Debug 

369 

370 @section pipe_tasks_calibrate_Purpose Description 

371 

372 Given an exposure with a good PSF model and aperture correction map 

373 (e.g. as provided by @ref characterizeImage::CharacterizeImageTask "CharacterizeImageTask"), 

374 perform the following operations: 

375 - Run detection and measurement 

376 - Run astrometry subtask to fit an improved WCS 

377 - Run photoCal subtask to fit the exposure's photometric zero-point 

378 

379 @section pipe_tasks_calibrate_Initialize Task initialisation 

380 

381 @copydoc \_\_init\_\_ 

382 

383 @section pipe_tasks_calibrate_IO Invoking the Task 

384 

385 If you want this task to unpersist inputs or persist outputs, then call 

386 the `runDataRef` method (a wrapper around the `run` method). 

387 

388 If you already have the inputs unpersisted and do not want to persist the 

389 output then it is more direct to call the `run` method: 

390 

391 @section pipe_tasks_calibrate_Config Configuration parameters 

392 

393 See @ref CalibrateConfig 

394 

395 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

396 

397 Exposure metadata 

398 <dl> 

399 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

400 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

401 task 

402 <dt>COLORTERM1 <dd>?? (always 0.0) 

403 <dt>COLORTERM2 <dd>?? (always 0.0) 

404 <dt>COLORTERM3 <dd>?? (always 0.0) 

405 </dl> 

406 

407 @section pipe_tasks_calibrate_Debug Debug variables 

408 

409 The command line task 

410 interface supports a flag 

411 `--debug` to import `debug.py` from your `$PYTHONPATH`; see 

412 <a href="https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a> 

413 for more about `debug.py`. 

414 

415 CalibrateTask has a debug dictionary containing one key: 

416 <dl> 

417 <dt>calibrate 

418 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

419 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

420 the meaning of the various symbols. 

421 </dl> 

422 

423 For example, put something like: 

424 @code{.py} 

425 import lsstDebug 

426 def DebugInfo(name): 

427 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

428 # call us recursively 

429 if name == "lsst.pipe.tasks.calibrate": 

430 di.display = dict( 

431 calibrate = 1, 

432 ) 

433 

434 return di 

435 

436 lsstDebug.Info = DebugInfo 

437 @endcode 

438 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

439 flag. 

440 

441 Some subtasks may have their own debug variables; see individual Task 

442 documentation. 

443 """ 

444 

445 # Example description used to live here, removed 2-20-2017 as per 

446 # https://jira.lsstcorp.org/browse/DM-9520 

447 

448 ConfigClass = CalibrateConfig 

449 _DefaultName = "calibrate" 

450 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

451 

452 def __init__(self, butler=None, astromRefObjLoader=None, 

453 photoRefObjLoader=None, icSourceSchema=None, 

454 initInputs=None, **kwargs): 

455 """!Construct a CalibrateTask 

456 

457 @param[in] butler The butler is passed to the refObjLoader constructor 

458 in case it is needed. Ignored if the refObjLoader argument 

459 provides a loader directly. 

460 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

461 that supplies an external reference catalog for astrometric 

462 calibration. May be None if the desired loader can be constructed 

463 from the butler argument or all steps requiring a reference catalog 

464 are disabled. 

465 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

466 that supplies an external reference catalog for photometric 

467 calibration. May be None if the desired loader can be constructed 

468 from the butler argument or all steps requiring a reference catalog 

469 are disabled. 

470 @param[in] icSourceSchema schema for icSource catalog, or None. 

471 Schema values specified in config.icSourceFieldsToCopy will be 

472 taken from this schema. If set to None, no values will be 

473 propagated from the icSourceCatalog 

474 @param[in,out] kwargs other keyword arguments for 

475 lsst.pipe.base.CmdLineTask 

476 """ 

477 super().__init__(**kwargs) 

478 

479 if icSourceSchema is None and butler is not None: 

480 # Use butler to read icSourceSchema from disk. 

481 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

482 

483 if icSourceSchema is None and butler is None and initInputs is not None: 

484 icSourceSchema = initInputs['icSourceSchema'].schema 

485 

486 if icSourceSchema is not None: 

487 # use a schema mapper to avoid copying each field separately 

488 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

489 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

490 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

491 

492 # Add fields to copy from an icSource catalog 

493 # and a field to indicate that the source matched a source in that 

494 # catalog. If any fields are missing then raise an exception, but 

495 # first find all missing fields in order to make the error message 

496 # more useful. 

497 self.calibSourceKey = self.schemaMapper.addOutputField( 

498 afwTable.Field["Flag"]("calib_detected", 

499 "Source was detected as an icSource")) 

500 missingFieldNames = [] 

501 for fieldName in self.config.icSourceFieldsToCopy: 

502 try: 

503 schemaItem = icSourceSchema.find(fieldName) 

504 except Exception: 

505 missingFieldNames.append(fieldName) 

506 else: 

507 # field found; if addMapping fails then raise an exception 

508 self.schemaMapper.addMapping(schemaItem.getKey()) 

509 

510 if missingFieldNames: 

511 raise RuntimeError("isSourceCat is missing fields {} " 

512 "specified in icSourceFieldsToCopy" 

513 .format(missingFieldNames)) 

514 

515 # produce a temporary schema to pass to the subtasks; finalize it 

516 # later 

517 self.schema = self.schemaMapper.editOutputSchema() 

518 else: 

519 self.schemaMapper = None 

520 self.schema = afwTable.SourceTable.makeMinimalSchema() 

521 self.makeSubtask('detection', schema=self.schema) 

522 

523 self.algMetadata = dafBase.PropertyList() 

524 

525 if self.config.doDeblend: 

526 self.makeSubtask("deblend", schema=self.schema) 

527 if self.config.doSkySources: 

528 self.makeSubtask("skySources") 

529 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

530 self.makeSubtask('measurement', schema=self.schema, 

531 algMetadata=self.algMetadata) 

532 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

533 algMetadata=self.algMetadata) 

534 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

535 if self.config.doApCorr: 

536 self.makeSubtask('applyApCorr', schema=self.schema) 

537 self.makeSubtask('catalogCalculation', schema=self.schema) 

538 

539 if self.config.doAstrometry: 

540 if astromRefObjLoader is None and butler is not None: 

541 self.makeSubtask('astromRefObjLoader', butler=butler) 

542 astromRefObjLoader = self.astromRefObjLoader 

543 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

544 schema=self.schema) 

545 if self.config.doPhotoCal: 

546 if photoRefObjLoader is None and butler is not None: 

547 self.makeSubtask('photoRefObjLoader', butler=butler) 

548 photoRefObjLoader = self.photoRefObjLoader 

549 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

550 schema=self.schema) 

551 if self.config.doComputeSummaryStats: 

552 self.makeSubtask('computeSummaryStats') 

553 

554 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

555 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

556 "reference object loaders.") 

557 

558 if self.schemaMapper is not None: 

559 # finalize the schema 

560 self.schema = self.schemaMapper.getOutputSchema() 

561 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

562 

563 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

564 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

565 self.outputSchema = sourceCatSchema 

566 

567 @timeMethod 

568 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

569 doUnpersist=True): 

570 """!Calibrate an exposure, optionally unpersisting inputs and 

571 persisting outputs. 

572 

573 This is a wrapper around the `run` method that unpersists inputs 

574 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

575 

576 @param[in] dataRef butler data reference corresponding to a science 

577 image 

578 @param[in,out] exposure characterized exposure (an 

579 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

580 icExp and icBackground. See `run` method for details of what is 

581 read and written. 

582 @param[in,out] background initial model of background already 

583 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

584 None if no background has been subtracted, though that is unusual 

585 for calibration. A refined background model is output. Ignored if 

586 exposure is None. 

587 @param[in] icSourceCat catalog from which to copy the fields specified 

588 by icSourceKeys, or None; 

589 @param[in] doUnpersist unpersist data: 

590 - if True, exposure, background and icSourceCat are read from 

591 dataRef and those three arguments must all be None; 

592 - if False the exposure must be provided; background and 

593 icSourceCat are optional. True is intended for running as a 

594 command-line task, False for running as a subtask 

595 @return same data as the calibrate method 

596 """ 

597 self.log.info("Processing %s", dataRef.dataId) 

598 

599 if doUnpersist: 

600 if any(item is not None for item in (exposure, background, 

601 icSourceCat)): 

602 raise RuntimeError("doUnpersist true; exposure, background " 

603 "and icSourceCat must all be None") 

604 exposure = dataRef.get("icExp", immediate=True) 

605 background = dataRef.get("icExpBackground", immediate=True) 

606 icSourceCat = dataRef.get("icSrc", immediate=True) 

607 elif exposure is None: 

608 raise RuntimeError("doUnpersist false; exposure must be provided") 

609 

610 exposureIdInfo = dataRef.get("expIdInfo") 

611 

612 calRes = self.run( 

613 exposure=exposure, 

614 exposureIdInfo=exposureIdInfo, 

615 background=background, 

616 icSourceCat=icSourceCat, 

617 ) 

618 

619 if self.config.doWrite: 

620 self.writeOutputs( 

621 dataRef=dataRef, 

622 exposure=calRes.exposure, 

623 background=calRes.background, 

624 sourceCat=calRes.sourceCat, 

625 astromMatches=calRes.astromMatches, 

626 matchMeta=calRes.matchMeta, 

627 ) 

628 

629 return calRes 

630 

631 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

632 inputs = butlerQC.get(inputRefs) 

633 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

634 

635 if self.config.doAstrometry: 

636 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

637 for ref in inputRefs.astromRefCat], 

638 refCats=inputs.pop('astromRefCat'), 

639 config=self.config.astromRefObjLoader, log=self.log) 

640 self.astrometry.setRefObjLoader(refObjLoader) 

641 

642 if self.config.doPhotoCal: 

643 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

644 for ref in inputRefs.photoRefCat], 

645 refCats=inputs.pop('photoRefCat'), 

646 config=self.config.photoRefObjLoader, 

647 log=self.log) 

648 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

649 

650 outputs = self.run(**inputs) 

651 

652 if self.config.doWriteMatches and self.config.doAstrometry: 

653 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

654 normalizedMatches.table.setMetadata(outputs.matchMeta) 

655 if self.config.doWriteMatchesDenormalized: 

656 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

657 outputs.matchesDenormalized = denormMatches 

658 outputs.matches = normalizedMatches 

659 butlerQC.put(outputs, outputRefs) 

660 

661 @timeMethod 

662 def run(self, exposure, exposureIdInfo=None, background=None, 

663 icSourceCat=None): 

664 """!Calibrate an exposure (science image or coadd) 

665 

666 @param[in,out] exposure exposure to calibrate (an 

667 lsst.afw.image.ExposureF or similar); 

668 in: 

669 - MaskedImage 

670 - Psf 

671 out: 

672 - MaskedImage has background subtracted 

673 - Wcs is replaced 

674 - PhotoCalib is replaced 

675 @param[in] exposureIdInfo ID info for exposure (an 

676 lsst.obs.base.ExposureIdInfo) If not provided, returned 

677 SourceCatalog IDs will not be globally unique. 

678 @param[in,out] background background model already subtracted from 

679 exposure (an lsst.afw.math.BackgroundList). May be None if no 

680 background has been subtracted, though that is unusual for 

681 calibration. A refined background model is output. 

682 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

683 from which we can copy some fields. 

684 

685 @return pipe_base Struct containing these fields: 

686 - exposure calibrate science exposure with refined WCS and PhotoCalib 

687 - background model of background subtracted from exposure (an 

688 lsst.afw.math.BackgroundList) 

689 - sourceCat catalog of measured sources 

690 - astromMatches list of source/refObj matches from the astrometry 

691 solver 

692 """ 

693 # detect, deblend and measure sources 

694 if exposureIdInfo is None: 

695 exposureIdInfo = ExposureIdInfo() 

696 

697 if background is None: 

698 background = BackgroundList() 

699 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

700 table = SourceTable.make(self.schema, sourceIdFactory) 

701 table.setMetadata(self.algMetadata) 

702 

703 detRes = self.detection.run(table=table, exposure=exposure, 

704 doSmooth=True) 

705 sourceCat = detRes.sources 

706 if detRes.fpSets.background: 

707 for bg in detRes.fpSets.background: 

708 background.append(bg) 

709 if self.config.doSkySources: 

710 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

711 if skySourceFootprints: 

712 for foot in skySourceFootprints: 

713 s = sourceCat.addNew() 

714 s.setFootprint(foot) 

715 s.set(self.skySourceKey, True) 

716 if self.config.doDeblend: 

717 self.deblend.run(exposure=exposure, sources=sourceCat) 

718 self.measurement.run( 

719 measCat=sourceCat, 

720 exposure=exposure, 

721 exposureId=exposureIdInfo.expId 

722 ) 

723 if self.config.doApCorr: 

724 self.applyApCorr.run( 

725 catalog=sourceCat, 

726 apCorrMap=exposure.getInfo().getApCorrMap() 

727 ) 

728 self.catalogCalculation.run(sourceCat) 

729 

730 self.setPrimaryFlags.run(sourceCat) 

731 

732 if icSourceCat is not None and \ 

733 len(self.config.icSourceFieldsToCopy) > 0: 

734 self.copyIcSourceFields(icSourceCat=icSourceCat, 

735 sourceCat=sourceCat) 

736 

737 # TODO DM-11568: this contiguous check-and-copy could go away if we 

738 # reserve enough space during SourceDetection and/or SourceDeblend. 

739 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

740 # contiguity now, so views are preserved from here on. 

741 if not sourceCat.isContiguous(): 

742 sourceCat = sourceCat.copy(deep=True) 

743 

744 # perform astrometry calibration: 

745 # fit an improved WCS and update the exposure's WCS in place 

746 astromMatches = None 

747 matchMeta = None 

748 if self.config.doAstrometry: 

749 try: 

750 astromRes = self.astrometry.run( 

751 exposure=exposure, 

752 sourceCat=sourceCat, 

753 ) 

754 astromMatches = astromRes.matches 

755 matchMeta = astromRes.matchMeta 

756 except Exception as e: 

757 if self.config.requireAstrometry: 

758 raise 

759 self.log.warning("Unable to perform astrometric calibration " 

760 "(%s): attempting to proceed", e) 

761 

762 # compute photometric calibration 

763 if self.config.doPhotoCal: 

764 try: 

765 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

766 exposure.setPhotoCalib(photoRes.photoCalib) 

767 # TODO: reword this to phrase it in terms of the calibration factor? 

768 self.log.info("Photometric zero-point: %f", 

769 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

770 self.setMetadata(exposure=exposure, photoRes=photoRes) 

771 except Exception as e: 

772 if self.config.requirePhotoCal: 

773 raise 

774 self.log.warning("Unable to perform photometric calibration " 

775 "(%s): attempting to proceed", e) 

776 self.setMetadata(exposure=exposure, photoRes=None) 

777 

778 self.postCalibrationMeasurement.run( 

779 measCat=sourceCat, 

780 exposure=exposure, 

781 exposureId=exposureIdInfo.expId 

782 ) 

783 

784 if self.config.doComputeSummaryStats: 

785 summary = self.computeSummaryStats.run(exposure=exposure, 

786 sources=sourceCat, 

787 background=background) 

788 exposure.getInfo().setSummaryStats(summary) 

789 

790 frame = getDebugFrame(self._display, "calibrate") 

791 if frame: 

792 displayAstrometry( 

793 sourceCat=sourceCat, 

794 exposure=exposure, 

795 matches=astromMatches, 

796 frame=frame, 

797 pause=False, 

798 ) 

799 

800 return pipeBase.Struct( 

801 exposure=exposure, 

802 background=background, 

803 sourceCat=sourceCat, 

804 astromMatches=astromMatches, 

805 matchMeta=matchMeta, 

806 # These are duplicate entries with different names for use with 

807 # gen3 middleware 

808 outputExposure=exposure, 

809 outputCat=sourceCat, 

810 outputBackground=background, 

811 ) 

812 

813 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

814 astromMatches, matchMeta): 

815 """Write output data to the output repository 

816 

817 @param[in] dataRef butler data reference corresponding to a science 

818 image 

819 @param[in] exposure exposure to write 

820 @param[in] background background model for exposure 

821 @param[in] sourceCat catalog of measured sources 

822 @param[in] astromMatches list of source/refObj matches from the 

823 astrometry solver 

824 """ 

825 dataRef.put(sourceCat, "src") 

826 if self.config.doWriteMatches and astromMatches is not None: 

827 normalizedMatches = afwTable.packMatches(astromMatches) 

828 normalizedMatches.table.setMetadata(matchMeta) 

829 dataRef.put(normalizedMatches, "srcMatch") 

830 if self.config.doWriteMatchesDenormalized: 

831 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

832 dataRef.put(denormMatches, "srcMatchFull") 

833 if self.config.doWriteExposure: 

834 dataRef.put(exposure, "calexp") 

835 dataRef.put(background, "calexpBackground") 

836 

837 def getSchemaCatalogs(self): 

838 """Return a dict of empty catalogs for each catalog dataset produced 

839 by this task. 

840 """ 

841 sourceCat = afwTable.SourceCatalog(self.schema) 

842 sourceCat.getTable().setMetadata(self.algMetadata) 

843 return {"src": sourceCat} 

844 

845 def setMetadata(self, exposure, photoRes=None): 

846 """!Set task and exposure metadata 

847 

848 Logs a warning and continues if needed data is missing. 

849 

850 @param[in,out] exposure exposure whose metadata is to be set 

851 @param[in] photoRes results of running photoCal; if None then it was 

852 not run 

853 """ 

854 if photoRes is None: 

855 return 

856 

857 metadata = exposure.getMetadata() 

858 

859 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

860 try: 

861 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

862 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

863 except Exception: 

864 self.log.warning("Could not set normalized MAGZERO in header: no " 

865 "exposure time") 

866 magZero = math.nan 

867 

868 try: 

869 metadata.set('MAGZERO', magZero) 

870 metadata.set('MAGZERO_RMS', photoRes.sigma) 

871 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

872 metadata.set('COLORTERM1', 0.0) 

873 metadata.set('COLORTERM2', 0.0) 

874 metadata.set('COLORTERM3', 0.0) 

875 except Exception as e: 

876 self.log.warning("Could not set exposure metadata: %s", e) 

877 

878 def copyIcSourceFields(self, icSourceCat, sourceCat): 

879 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

880 

881 @param[in] icSourceCat catalog from which to copy fields 

882 @param[in,out] sourceCat catalog to which to copy fields 

883 

884 The fields copied are those specified by `config.icSourceFieldsToCopy` 

885 that actually exist in the schema. This was set up by the constructor 

886 using self.schemaMapper. 

887 """ 

888 if self.schemaMapper is None: 

889 raise RuntimeError("To copy icSource fields you must specify " 

890 "icSourceSchema nd icSourceKeys when " 

891 "constructing this task") 

892 if icSourceCat is None or sourceCat is None: 

893 raise RuntimeError("icSourceCat and sourceCat must both be " 

894 "specified") 

895 if len(self.config.icSourceFieldsToCopy) == 0: 

896 self.log.warning("copyIcSourceFields doing nothing because " 

897 "icSourceFieldsToCopy is empty") 

898 return 

899 

900 mc = afwTable.MatchControl() 

901 mc.findOnlyClosest = False # return all matched objects 

902 matches = afwTable.matchXy(icSourceCat, sourceCat, 

903 self.config.matchRadiusPix, mc) 

904 if self.config.doDeblend: 

905 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

906 # if deblended, keep children 

907 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

908 

909 # Because we had to allow multiple matches to handle parents, we now 

910 # need to prune to the best matches 

911 # closest matches as a dict of icSourceCat source ID: 

912 # (icSourceCat source, sourceCat source, distance in pixels) 

913 bestMatches = {} 

914 for m0, m1, d in matches: 

915 id0 = m0.getId() 

916 match = bestMatches.get(id0) 

917 if match is None or d <= match[2]: 

918 bestMatches[id0] = (m0, m1, d) 

919 matches = list(bestMatches.values()) 

920 

921 # Check that no sourceCat sources are listed twice (we already know 

922 # that each match has a unique icSourceCat source ID, due to using 

923 # that ID as the key in bestMatches) 

924 numMatches = len(matches) 

925 numUniqueSources = len(set(m[1].getId() for m in matches)) 

926 if numUniqueSources != numMatches: 

927 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

928 "sources", numMatches, numUniqueSources) 

929 

930 self.log.info("Copying flags from icSourceCat to sourceCat for " 

931 "%d sources", numMatches) 

932 

933 # For each match: set the calibSourceKey flag and copy the desired 

934 # fields 

935 for icSrc, src, d in matches: 

936 src.setFlag(self.calibSourceKey, True) 

937 # src.assign copies the footprint from icSrc, which we don't want 

938 # (DM-407) 

939 # so set icSrc's footprint to src's footprint before src.assign, 

940 # then restore it 

941 icSrcFootprint = icSrc.getFootprint() 

942 try: 

943 icSrc.setFootprint(src.getFootprint()) 

944 src.assign(icSrc, self.schemaMapper) 

945 finally: 

946 icSrc.setFootprint(icSrcFootprint)