Coverage for python/lsst/pipe/tasks/calibrate.py: 20%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

316 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from lsst.utils.timer import timeMethod 

41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

42from .fakes import BaseFakeSourcesTask 

43from .photoCal import PhotoCalTask 

44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

45 

46 

47__all__ = ["CalibrateConfig", "CalibrateTask"] 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="cal_ref_cat", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="cal_ref_cat", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask""" 

155 doWrite = pexConfig.Field( 

156 dtype=bool, 

157 default=True, 

158 doc="Save calibration results?", 

159 ) 

160 doWriteHeavyFootprintsInSources = pexConfig.Field( 

161 dtype=bool, 

162 default=True, 

163 doc="Include HeavyFootprint data in source table? If false then heavy " 

164 "footprints are saved as normal footprints, which saves some space" 

165 ) 

166 doWriteMatches = pexConfig.Field( 

167 dtype=bool, 

168 default=True, 

169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

170 ) 

171 doWriteMatchesDenormalized = pexConfig.Field( 

172 dtype=bool, 

173 default=False, 

174 doc=("Write reference matches in denormalized format? " 

175 "This format uses more disk space, but is more convenient to " 

176 "read. Ignored if doWriteMatches=False or doWrite=False."), 

177 ) 

178 doAstrometry = pexConfig.Field( 

179 dtype=bool, 

180 default=True, 

181 doc="Perform astrometric calibration?", 

182 ) 

183 astromRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for astrometric calibration", 

186 ) 

187 photoRefObjLoader = pexConfig.ConfigurableField( 

188 target=LoadIndexedReferenceObjectsTask, 

189 doc="reference object loader for photometric calibration", 

190 ) 

191 astrometry = pexConfig.ConfigurableField( 

192 target=AstrometryTask, 

193 doc="Perform astrometric calibration to refine the WCS", 

194 ) 

195 requireAstrometry = pexConfig.Field( 

196 dtype=bool, 

197 default=True, 

198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

199 "false."), 

200 ) 

201 doPhotoCal = pexConfig.Field( 

202 dtype=bool, 

203 default=True, 

204 doc="Perform phometric calibration?", 

205 ) 

206 requirePhotoCal = pexConfig.Field( 

207 dtype=bool, 

208 default=True, 

209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

210 "false."), 

211 ) 

212 photoCal = pexConfig.ConfigurableField( 

213 target=PhotoCalTask, 

214 doc="Perform photometric calibration", 

215 ) 

216 icSourceFieldsToCopy = pexConfig.ListField( 

217 dtype=str, 

218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

219 doc=("Fields to copy from the icSource catalog to the output catalog " 

220 "for matching sources Any missing fields will trigger a " 

221 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

222 ) 

223 matchRadiusPix = pexConfig.Field( 

224 dtype=float, 

225 default=3, 

226 doc=("Match radius for matching icSourceCat objects to sourceCat " 

227 "objects (pixels)"), 

228 ) 

229 checkUnitsParseStrict = pexConfig.Field( 

230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

231 "'warn' or 'silent'"), 

232 dtype=str, 

233 default="raise", 

234 ) 

235 detection = pexConfig.ConfigurableField( 

236 target=SourceDetectionTask, 

237 doc="Detect sources" 

238 ) 

239 doDeblend = pexConfig.Field( 

240 dtype=bool, 

241 default=True, 

242 doc="Run deblender input exposure" 

243 ) 

244 deblend = pexConfig.ConfigurableField( 

245 target=SourceDeblendTask, 

246 doc="Split blended sources into their components" 

247 ) 

248 doSkySources = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Generate sky sources?", 

252 ) 

253 skySources = pexConfig.ConfigurableField( 

254 target=SkyObjectsTask, 

255 doc="Generate sky sources", 

256 ) 

257 measurement = pexConfig.ConfigurableField( 

258 target=SingleFrameMeasurementTask, 

259 doc="Measure sources" 

260 ) 

261 postCalibrationMeasurement = pexConfig.ConfigurableField( 

262 target=SingleFrameMeasurementTask, 

263 doc="Second round of measurement for plugins that need to be run after photocal" 

264 ) 

265 setPrimaryFlags = pexConfig.ConfigurableField( 

266 target=SetPrimaryFlagsTask, 

267 doc=("Set flags for primary source classification in single frame " 

268 "processing. True if sources are not sky sources and not a parent.") 

269 ) 

270 doApCorr = pexConfig.Field( 

271 dtype=bool, 

272 default=True, 

273 doc="Run subtask to apply aperture correction" 

274 ) 

275 applyApCorr = pexConfig.ConfigurableField( 

276 target=ApplyApCorrTask, 

277 doc="Subtask to apply aperture corrections" 

278 ) 

279 # If doApCorr is False, and the exposure does not have apcorrections 

280 # already applied, the active plugins in catalogCalculation almost 

281 # certainly should not contain the characterization plugin 

282 catalogCalculation = pexConfig.ConfigurableField( 

283 target=CatalogCalculationTask, 

284 doc="Subtask to run catalogCalculation plugins on catalog" 

285 ) 

286 doInsertFakes = pexConfig.Field( 

287 dtype=bool, 

288 default=False, 

289 doc="Run fake sources injection task", 

290 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. " 

291 "Please use ProcessCcdWithFakesTask instead.") 

292 ) 

293 insertFakes = pexConfig.ConfigurableField( 

294 target=BaseFakeSourcesTask, 

295 doc="Injection of fake sources for testing purposes (must be " 

296 "retargeted)", 

297 deprecated=("insertFakes is no longer supported. This config will be removed after v24. " 

298 "Please use ProcessCcdWithFakesTask instead.") 

299 ) 

300 doComputeSummaryStats = pexConfig.Field( 

301 dtype=bool, 

302 default=True, 

303 doc="Run subtask to measure exposure summary statistics?" 

304 ) 

305 computeSummaryStats = pexConfig.ConfigurableField( 

306 target=ComputeExposureSummaryStatsTask, 

307 doc="Subtask to run computeSummaryStats on exposure" 

308 ) 

309 doWriteExposure = pexConfig.Field( 

310 dtype=bool, 

311 default=True, 

312 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

313 "normal calexp but as a fakes_calexp." 

314 ) 

315 

316 def setDefaults(self): 

317 super().setDefaults() 

318 self.detection.doTempLocalBackground = False 

319 self.deblend.maxFootprintSize = 2000 

320 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

321 self.postCalibrationMeasurement.doReplaceWithNoise = False 

322 for key in self.postCalibrationMeasurement.slots: 

323 setattr(self.postCalibrationMeasurement.slots, key, None) 

324 

325 def validate(self): 

326 super().validate() 

327 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

328 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

329 raise ValueError( 

330 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

331 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

332 ) 

333 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

334 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

335 raise ValueError( 

336 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

337 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

338 ) 

339 

340 

341## \addtogroup LSST_task_documentation 

342## \{ 

343## \page page_CalibrateTask CalibrateTask 

344## \ref CalibrateTask_ "CalibrateTask" 

345## \copybrief CalibrateTask 

346## \} 

347 

348class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

349 r"""!Calibrate an exposure: measure sources and perform astrometric and 

350 photometric calibration 

351 

352 @anchor CalibrateTask_ 

353 

354 @section pipe_tasks_calibrate_Contents Contents 

355 

356 - @ref pipe_tasks_calibrate_Purpose 

357 - @ref pipe_tasks_calibrate_Initialize 

358 - @ref pipe_tasks_calibrate_IO 

359 - @ref pipe_tasks_calibrate_Config 

360 - @ref pipe_tasks_calibrate_Metadata 

361 - @ref pipe_tasks_calibrate_Debug 

362 

363 @section pipe_tasks_calibrate_Purpose Description 

364 

365 Given an exposure with a good PSF model and aperture correction map 

366 (e.g. as provided by @ref characterizeImage::CharacterizeImageTask "CharacterizeImageTask"), 

367 perform the following operations: 

368 - Run detection and measurement 

369 - Run astrometry subtask to fit an improved WCS 

370 - Run photoCal subtask to fit the exposure's photometric zero-point 

371 

372 @section pipe_tasks_calibrate_Initialize Task initialisation 

373 

374 @copydoc \_\_init\_\_ 

375 

376 @section pipe_tasks_calibrate_IO Invoking the Task 

377 

378 If you want this task to unpersist inputs or persist outputs, then call 

379 the `runDataRef` method (a wrapper around the `run` method). 

380 

381 If you already have the inputs unpersisted and do not want to persist the 

382 output then it is more direct to call the `run` method: 

383 

384 @section pipe_tasks_calibrate_Config Configuration parameters 

385 

386 See @ref CalibrateConfig 

387 

388 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

389 

390 Exposure metadata 

391 <dl> 

392 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

393 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

394 task 

395 <dt>COLORTERM1 <dd>?? (always 0.0) 

396 <dt>COLORTERM2 <dd>?? (always 0.0) 

397 <dt>COLORTERM3 <dd>?? (always 0.0) 

398 </dl> 

399 

400 @section pipe_tasks_calibrate_Debug Debug variables 

401 

402 The command line task 

403 interface supports a flag 

404 `--debug` to import `debug.py` from your `$PYTHONPATH`; see 

405 <a href="https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a> 

406 for more about `debug.py`. 

407 

408 CalibrateTask has a debug dictionary containing one key: 

409 <dl> 

410 <dt>calibrate 

411 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

412 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

413 the meaning of the various symbols. 

414 </dl> 

415 

416 For example, put something like: 

417 @code{.py} 

418 import lsstDebug 

419 def DebugInfo(name): 

420 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

421 # call us recursively 

422 if name == "lsst.pipe.tasks.calibrate": 

423 di.display = dict( 

424 calibrate = 1, 

425 ) 

426 

427 return di 

428 

429 lsstDebug.Info = DebugInfo 

430 @endcode 

431 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

432 flag. 

433 

434 Some subtasks may have their own debug variables; see individual Task 

435 documentation. 

436 """ 

437 

438 # Example description used to live here, removed 2-20-2017 as per 

439 # https://jira.lsstcorp.org/browse/DM-9520 

440 

441 ConfigClass = CalibrateConfig 

442 _DefaultName = "calibrate" 

443 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

444 

445 def __init__(self, butler=None, astromRefObjLoader=None, 

446 photoRefObjLoader=None, icSourceSchema=None, 

447 initInputs=None, **kwargs): 

448 """!Construct a CalibrateTask 

449 

450 @param[in] butler The butler is passed to the refObjLoader constructor 

451 in case it is needed. Ignored if the refObjLoader argument 

452 provides a loader directly. 

453 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

454 that supplies an external reference catalog for astrometric 

455 calibration. May be None if the desired loader can be constructed 

456 from the butler argument or all steps requiring a reference catalog 

457 are disabled. 

458 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

459 that supplies an external reference catalog for photometric 

460 calibration. May be None if the desired loader can be constructed 

461 from the butler argument or all steps requiring a reference catalog 

462 are disabled. 

463 @param[in] icSourceSchema schema for icSource catalog, or None. 

464 Schema values specified in config.icSourceFieldsToCopy will be 

465 taken from this schema. If set to None, no values will be 

466 propagated from the icSourceCatalog 

467 @param[in,out] kwargs other keyword arguments for 

468 lsst.pipe.base.CmdLineTask 

469 """ 

470 super().__init__(**kwargs) 

471 

472 if icSourceSchema is None and butler is not None: 

473 # Use butler to read icSourceSchema from disk. 

474 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

475 

476 if icSourceSchema is None and butler is None and initInputs is not None: 

477 icSourceSchema = initInputs['icSourceSchema'].schema 

478 

479 if icSourceSchema is not None: 

480 # use a schema mapper to avoid copying each field separately 

481 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

482 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

483 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

484 

485 # Add fields to copy from an icSource catalog 

486 # and a field to indicate that the source matched a source in that 

487 # catalog. If any fields are missing then raise an exception, but 

488 # first find all missing fields in order to make the error message 

489 # more useful. 

490 self.calibSourceKey = self.schemaMapper.addOutputField( 

491 afwTable.Field["Flag"]("calib_detected", 

492 "Source was detected as an icSource")) 

493 missingFieldNames = [] 

494 for fieldName in self.config.icSourceFieldsToCopy: 

495 try: 

496 schemaItem = icSourceSchema.find(fieldName) 

497 except Exception: 

498 missingFieldNames.append(fieldName) 

499 else: 

500 # field found; if addMapping fails then raise an exception 

501 self.schemaMapper.addMapping(schemaItem.getKey()) 

502 

503 if missingFieldNames: 

504 raise RuntimeError("isSourceCat is missing fields {} " 

505 "specified in icSourceFieldsToCopy" 

506 .format(missingFieldNames)) 

507 

508 # produce a temporary schema to pass to the subtasks; finalize it 

509 # later 

510 self.schema = self.schemaMapper.editOutputSchema() 

511 else: 

512 self.schemaMapper = None 

513 self.schema = afwTable.SourceTable.makeMinimalSchema() 

514 self.makeSubtask('detection', schema=self.schema) 

515 

516 self.algMetadata = dafBase.PropertyList() 

517 

518 if self.config.doDeblend: 

519 self.makeSubtask("deblend", schema=self.schema) 

520 if self.config.doSkySources: 

521 self.makeSubtask("skySources") 

522 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

523 self.makeSubtask('measurement', schema=self.schema, 

524 algMetadata=self.algMetadata) 

525 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

526 algMetadata=self.algMetadata) 

527 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

528 if self.config.doApCorr: 

529 self.makeSubtask('applyApCorr', schema=self.schema) 

530 self.makeSubtask('catalogCalculation', schema=self.schema) 

531 

532 if self.config.doAstrometry: 

533 if astromRefObjLoader is None and butler is not None: 

534 self.makeSubtask('astromRefObjLoader', butler=butler) 

535 astromRefObjLoader = self.astromRefObjLoader 

536 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

537 schema=self.schema) 

538 if self.config.doPhotoCal: 

539 if photoRefObjLoader is None and butler is not None: 

540 self.makeSubtask('photoRefObjLoader', butler=butler) 

541 photoRefObjLoader = self.photoRefObjLoader 

542 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

543 schema=self.schema) 

544 if self.config.doComputeSummaryStats: 

545 self.makeSubtask('computeSummaryStats') 

546 

547 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

548 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

549 "reference object loaders.") 

550 

551 if self.schemaMapper is not None: 

552 # finalize the schema 

553 self.schema = self.schemaMapper.getOutputSchema() 

554 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

555 

556 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

557 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

558 self.outputSchema = sourceCatSchema 

559 

560 @timeMethod 

561 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

562 doUnpersist=True): 

563 """!Calibrate an exposure, optionally unpersisting inputs and 

564 persisting outputs. 

565 

566 This is a wrapper around the `run` method that unpersists inputs 

567 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

568 

569 @param[in] dataRef butler data reference corresponding to a science 

570 image 

571 @param[in,out] exposure characterized exposure (an 

572 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

573 icExp and icBackground. See `run` method for details of what is 

574 read and written. 

575 @param[in,out] background initial model of background already 

576 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

577 None if no background has been subtracted, though that is unusual 

578 for calibration. A refined background model is output. Ignored if 

579 exposure is None. 

580 @param[in] icSourceCat catalog from which to copy the fields specified 

581 by icSourceKeys, or None; 

582 @param[in] doUnpersist unpersist data: 

583 - if True, exposure, background and icSourceCat are read from 

584 dataRef and those three arguments must all be None; 

585 - if False the exposure must be provided; background and 

586 icSourceCat are optional. True is intended for running as a 

587 command-line task, False for running as a subtask 

588 @return same data as the calibrate method 

589 """ 

590 self.log.info("Processing %s", dataRef.dataId) 

591 

592 if doUnpersist: 

593 if any(item is not None for item in (exposure, background, 

594 icSourceCat)): 

595 raise RuntimeError("doUnpersist true; exposure, background " 

596 "and icSourceCat must all be None") 

597 exposure = dataRef.get("icExp", immediate=True) 

598 background = dataRef.get("icExpBackground", immediate=True) 

599 icSourceCat = dataRef.get("icSrc", immediate=True) 

600 elif exposure is None: 

601 raise RuntimeError("doUnpersist false; exposure must be provided") 

602 

603 exposureIdInfo = dataRef.get("expIdInfo") 

604 

605 calRes = self.run( 

606 exposure=exposure, 

607 exposureIdInfo=exposureIdInfo, 

608 background=background, 

609 icSourceCat=icSourceCat, 

610 ) 

611 

612 if self.config.doWrite: 

613 self.writeOutputs( 

614 dataRef=dataRef, 

615 exposure=calRes.exposure, 

616 background=calRes.background, 

617 sourceCat=calRes.sourceCat, 

618 astromMatches=calRes.astromMatches, 

619 matchMeta=calRes.matchMeta, 

620 ) 

621 

622 return calRes 

623 

624 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

625 inputs = butlerQC.get(inputRefs) 

626 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

627 

628 if self.config.doAstrometry: 

629 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

630 for ref in inputRefs.astromRefCat], 

631 refCats=inputs.pop('astromRefCat'), 

632 config=self.config.astromRefObjLoader, log=self.log) 

633 self.astrometry.setRefObjLoader(refObjLoader) 

634 

635 if self.config.doPhotoCal: 

636 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

637 for ref in inputRefs.photoRefCat], 

638 refCats=inputs.pop('photoRefCat'), 

639 config=self.config.photoRefObjLoader, 

640 log=self.log) 

641 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

642 

643 outputs = self.run(**inputs) 

644 

645 if self.config.doWriteMatches and self.config.doAstrometry: 

646 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

647 normalizedMatches.table.setMetadata(outputs.matchMeta) 

648 if self.config.doWriteMatchesDenormalized: 

649 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

650 outputs.matchesDenormalized = denormMatches 

651 outputs.matches = normalizedMatches 

652 butlerQC.put(outputs, outputRefs) 

653 

654 @timeMethod 

655 def run(self, exposure, exposureIdInfo=None, background=None, 

656 icSourceCat=None): 

657 """!Calibrate an exposure (science image or coadd) 

658 

659 @param[in,out] exposure exposure to calibrate (an 

660 lsst.afw.image.ExposureF or similar); 

661 in: 

662 - MaskedImage 

663 - Psf 

664 out: 

665 - MaskedImage has background subtracted 

666 - Wcs is replaced 

667 - PhotoCalib is replaced 

668 @param[in] exposureIdInfo ID info for exposure (an 

669 lsst.obs.base.ExposureIdInfo) If not provided, returned 

670 SourceCatalog IDs will not be globally unique. 

671 @param[in,out] background background model already subtracted from 

672 exposure (an lsst.afw.math.BackgroundList). May be None if no 

673 background has been subtracted, though that is unusual for 

674 calibration. A refined background model is output. 

675 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

676 from which we can copy some fields. 

677 

678 @return pipe_base Struct containing these fields: 

679 - exposure calibrate science exposure with refined WCS and PhotoCalib 

680 - background model of background subtracted from exposure (an 

681 lsst.afw.math.BackgroundList) 

682 - sourceCat catalog of measured sources 

683 - astromMatches list of source/refObj matches from the astrometry 

684 solver 

685 """ 

686 # detect, deblend and measure sources 

687 if exposureIdInfo is None: 

688 exposureIdInfo = ExposureIdInfo() 

689 

690 if background is None: 

691 background = BackgroundList() 

692 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

693 table = SourceTable.make(self.schema, sourceIdFactory) 

694 table.setMetadata(self.algMetadata) 

695 

696 detRes = self.detection.run(table=table, exposure=exposure, 

697 doSmooth=True) 

698 sourceCat = detRes.sources 

699 if detRes.fpSets.background: 

700 for bg in detRes.fpSets.background: 

701 background.append(bg) 

702 if self.config.doSkySources: 

703 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

704 if skySourceFootprints: 

705 for foot in skySourceFootprints: 

706 s = sourceCat.addNew() 

707 s.setFootprint(foot) 

708 s.set(self.skySourceKey, True) 

709 if self.config.doDeblend: 

710 self.deblend.run(exposure=exposure, sources=sourceCat) 

711 self.measurement.run( 

712 measCat=sourceCat, 

713 exposure=exposure, 

714 exposureId=exposureIdInfo.expId 

715 ) 

716 if self.config.doApCorr: 

717 self.applyApCorr.run( 

718 catalog=sourceCat, 

719 apCorrMap=exposure.getInfo().getApCorrMap() 

720 ) 

721 self.catalogCalculation.run(sourceCat) 

722 

723 self.setPrimaryFlags.run(sourceCat) 

724 

725 if icSourceCat is not None and \ 

726 len(self.config.icSourceFieldsToCopy) > 0: 

727 self.copyIcSourceFields(icSourceCat=icSourceCat, 

728 sourceCat=sourceCat) 

729 

730 # TODO DM-11568: this contiguous check-and-copy could go away if we 

731 # reserve enough space during SourceDetection and/or SourceDeblend. 

732 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

733 # contiguity now, so views are preserved from here on. 

734 if not sourceCat.isContiguous(): 

735 sourceCat = sourceCat.copy(deep=True) 

736 

737 # perform astrometry calibration: 

738 # fit an improved WCS and update the exposure's WCS in place 

739 astromMatches = None 

740 matchMeta = None 

741 if self.config.doAstrometry: 

742 try: 

743 astromRes = self.astrometry.run( 

744 exposure=exposure, 

745 sourceCat=sourceCat, 

746 ) 

747 astromMatches = astromRes.matches 

748 matchMeta = astromRes.matchMeta 

749 except Exception as e: 

750 if self.config.requireAstrometry: 

751 raise 

752 self.log.warning("Unable to perform astrometric calibration " 

753 "(%s): attempting to proceed", e) 

754 

755 # compute photometric calibration 

756 if self.config.doPhotoCal: 

757 try: 

758 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

759 exposure.setPhotoCalib(photoRes.photoCalib) 

760 # TODO: reword this to phrase it in terms of the calibration factor? 

761 self.log.info("Photometric zero-point: %f", 

762 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

763 self.setMetadata(exposure=exposure, photoRes=photoRes) 

764 except Exception as e: 

765 if self.config.requirePhotoCal: 

766 raise 

767 self.log.warning("Unable to perform photometric calibration " 

768 "(%s): attempting to proceed", e) 

769 self.setMetadata(exposure=exposure, photoRes=None) 

770 

771 self.postCalibrationMeasurement.run( 

772 measCat=sourceCat, 

773 exposure=exposure, 

774 exposureId=exposureIdInfo.expId 

775 ) 

776 

777 if self.config.doComputeSummaryStats: 

778 summary = self.computeSummaryStats.run(exposure=exposure, 

779 sources=sourceCat, 

780 background=background) 

781 exposure.getInfo().setSummaryStats(summary) 

782 

783 frame = getDebugFrame(self._display, "calibrate") 

784 if frame: 

785 displayAstrometry( 

786 sourceCat=sourceCat, 

787 exposure=exposure, 

788 matches=astromMatches, 

789 frame=frame, 

790 pause=False, 

791 ) 

792 

793 return pipeBase.Struct( 

794 exposure=exposure, 

795 background=background, 

796 sourceCat=sourceCat, 

797 astromMatches=astromMatches, 

798 matchMeta=matchMeta, 

799 # These are duplicate entries with different names for use with 

800 # gen3 middleware 

801 outputExposure=exposure, 

802 outputCat=sourceCat, 

803 outputBackground=background, 

804 ) 

805 

806 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

807 astromMatches, matchMeta): 

808 """Write output data to the output repository 

809 

810 @param[in] dataRef butler data reference corresponding to a science 

811 image 

812 @param[in] exposure exposure to write 

813 @param[in] background background model for exposure 

814 @param[in] sourceCat catalog of measured sources 

815 @param[in] astromMatches list of source/refObj matches from the 

816 astrometry solver 

817 """ 

818 dataRef.put(sourceCat, "src") 

819 if self.config.doWriteMatches and astromMatches is not None: 

820 normalizedMatches = afwTable.packMatches(astromMatches) 

821 normalizedMatches.table.setMetadata(matchMeta) 

822 dataRef.put(normalizedMatches, "srcMatch") 

823 if self.config.doWriteMatchesDenormalized: 

824 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

825 dataRef.put(denormMatches, "srcMatchFull") 

826 if self.config.doWriteExposure: 

827 dataRef.put(exposure, "calexp") 

828 dataRef.put(background, "calexpBackground") 

829 

830 def getSchemaCatalogs(self): 

831 """Return a dict of empty catalogs for each catalog dataset produced 

832 by this task. 

833 """ 

834 sourceCat = afwTable.SourceCatalog(self.schema) 

835 sourceCat.getTable().setMetadata(self.algMetadata) 

836 return {"src": sourceCat} 

837 

838 def setMetadata(self, exposure, photoRes=None): 

839 """!Set task and exposure metadata 

840 

841 Logs a warning and continues if needed data is missing. 

842 

843 @param[in,out] exposure exposure whose metadata is to be set 

844 @param[in] photoRes results of running photoCal; if None then it was 

845 not run 

846 """ 

847 if photoRes is None: 

848 return 

849 

850 metadata = exposure.getMetadata() 

851 

852 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

853 try: 

854 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

855 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

856 except Exception: 

857 self.log.warning("Could not set normalized MAGZERO in header: no " 

858 "exposure time") 

859 magZero = math.nan 

860 

861 try: 

862 metadata.set('MAGZERO', magZero) 

863 metadata.set('MAGZERO_RMS', photoRes.sigma) 

864 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

865 metadata.set('COLORTERM1', 0.0) 

866 metadata.set('COLORTERM2', 0.0) 

867 metadata.set('COLORTERM3', 0.0) 

868 except Exception as e: 

869 self.log.warning("Could not set exposure metadata: %s", e) 

870 

871 def copyIcSourceFields(self, icSourceCat, sourceCat): 

872 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

873 

874 @param[in] icSourceCat catalog from which to copy fields 

875 @param[in,out] sourceCat catalog to which to copy fields 

876 

877 The fields copied are those specified by `config.icSourceFieldsToCopy` 

878 that actually exist in the schema. This was set up by the constructor 

879 using self.schemaMapper. 

880 """ 

881 if self.schemaMapper is None: 

882 raise RuntimeError("To copy icSource fields you must specify " 

883 "icSourceSchema nd icSourceKeys when " 

884 "constructing this task") 

885 if icSourceCat is None or sourceCat is None: 

886 raise RuntimeError("icSourceCat and sourceCat must both be " 

887 "specified") 

888 if len(self.config.icSourceFieldsToCopy) == 0: 

889 self.log.warning("copyIcSourceFields doing nothing because " 

890 "icSourceFieldsToCopy is empty") 

891 return 

892 

893 mc = afwTable.MatchControl() 

894 mc.findOnlyClosest = False # return all matched objects 

895 matches = afwTable.matchXy(icSourceCat, sourceCat, 

896 self.config.matchRadiusPix, mc) 

897 if self.config.doDeblend: 

898 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

899 # if deblended, keep children 

900 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

901 

902 # Because we had to allow multiple matches to handle parents, we now 

903 # need to prune to the best matches 

904 # closest matches as a dict of icSourceCat source ID: 

905 # (icSourceCat source, sourceCat source, distance in pixels) 

906 bestMatches = {} 

907 for m0, m1, d in matches: 

908 id0 = m0.getId() 

909 match = bestMatches.get(id0) 

910 if match is None or d <= match[2]: 

911 bestMatches[id0] = (m0, m1, d) 

912 matches = list(bestMatches.values()) 

913 

914 # Check that no sourceCat sources are listed twice (we already know 

915 # that each match has a unique icSourceCat source ID, due to using 

916 # that ID as the key in bestMatches) 

917 numMatches = len(matches) 

918 numUniqueSources = len(set(m[1].getId() for m in matches)) 

919 if numUniqueSources != numMatches: 

920 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

921 "sources", numMatches, numUniqueSources) 

922 

923 self.log.info("Copying flags from icSourceCat to sourceCat for " 

924 "%d sources", numMatches) 

925 

926 # For each match: set the calibSourceKey flag and copy the desired 

927 # fields 

928 for icSrc, src, d in matches: 

929 src.setFlag(self.calibSourceKey, True) 

930 # src.assign copies the footprint from icSrc, which we don't want 

931 # (DM-407) 

932 # so set icSrc's footprint to src's footprint before src.assign, 

933 # then restore it 

934 icSrcFootprint = icSrc.getFootprint() 

935 try: 

936 icSrc.setFootprint(src.getFootprint()) 

937 src.assign(icSrc, self.schemaMapper) 

938 finally: 

939 icSrc.setFootprint(icSrcFootprint)