Coverage for python/lsst/pipe/tasks/calibrate.py: 18%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

329 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from lsst.utils.timer import timeMethod 

41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

42from .fakes import BaseFakeSourcesTask 

43from .photoCal import PhotoCalTask 

44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

45 

46 

47__all__ = ["CalibrateConfig", "CalibrateTask"] 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="cal_ref_cat", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="cal_ref_cat", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask""" 

155 doWrite = pexConfig.Field( 

156 dtype=bool, 

157 default=True, 

158 doc="Save calibration results?", 

159 ) 

160 doWriteHeavyFootprintsInSources = pexConfig.Field( 

161 dtype=bool, 

162 default=True, 

163 doc="Include HeavyFootprint data in source table? If false then heavy " 

164 "footprints are saved as normal footprints, which saves some space" 

165 ) 

166 doWriteMatches = pexConfig.Field( 

167 dtype=bool, 

168 default=True, 

169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

170 ) 

171 doWriteMatchesDenormalized = pexConfig.Field( 

172 dtype=bool, 

173 default=False, 

174 doc=("Write reference matches in denormalized format? " 

175 "This format uses more disk space, but is more convenient to " 

176 "read. Ignored if doWriteMatches=False or doWrite=False."), 

177 ) 

178 doAstrometry = pexConfig.Field( 

179 dtype=bool, 

180 default=True, 

181 doc="Perform astrometric calibration?", 

182 ) 

183 astromRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for astrometric calibration", 

186 ) 

187 photoRefObjLoader = pexConfig.ConfigurableField( 

188 target=LoadIndexedReferenceObjectsTask, 

189 doc="reference object loader for photometric calibration", 

190 ) 

191 astrometry = pexConfig.ConfigurableField( 

192 target=AstrometryTask, 

193 doc="Perform astrometric calibration to refine the WCS", 

194 ) 

195 requireAstrometry = pexConfig.Field( 

196 dtype=bool, 

197 default=True, 

198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

199 "false."), 

200 ) 

201 doPhotoCal = pexConfig.Field( 

202 dtype=bool, 

203 default=True, 

204 doc="Perform phometric calibration?", 

205 ) 

206 requirePhotoCal = pexConfig.Field( 

207 dtype=bool, 

208 default=True, 

209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

210 "false."), 

211 ) 

212 photoCal = pexConfig.ConfigurableField( 

213 target=PhotoCalTask, 

214 doc="Perform photometric calibration", 

215 ) 

216 icSourceFieldsToCopy = pexConfig.ListField( 

217 dtype=str, 

218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

219 doc=("Fields to copy from the icSource catalog to the output catalog " 

220 "for matching sources Any missing fields will trigger a " 

221 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

222 ) 

223 matchRadiusPix = pexConfig.Field( 

224 dtype=float, 

225 default=3, 

226 doc=("Match radius for matching icSourceCat objects to sourceCat " 

227 "objects (pixels)"), 

228 ) 

229 checkUnitsParseStrict = pexConfig.Field( 

230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

231 "'warn' or 'silent'"), 

232 dtype=str, 

233 default="raise", 

234 ) 

235 detection = pexConfig.ConfigurableField( 

236 target=SourceDetectionTask, 

237 doc="Detect sources" 

238 ) 

239 doDeblend = pexConfig.Field( 

240 dtype=bool, 

241 default=True, 

242 doc="Run deblender input exposure" 

243 ) 

244 deblend = pexConfig.ConfigurableField( 

245 target=SourceDeblendTask, 

246 doc="Split blended sources into their components" 

247 ) 

248 doSkySources = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Generate sky sources?", 

252 ) 

253 skySources = pexConfig.ConfigurableField( 

254 target=SkyObjectsTask, 

255 doc="Generate sky sources", 

256 ) 

257 measurement = pexConfig.ConfigurableField( 

258 target=SingleFrameMeasurementTask, 

259 doc="Measure sources" 

260 ) 

261 setPrimaryFlags = pexConfig.ConfigurableField( 

262 target=SetPrimaryFlagsTask, 

263 doc=("Set flags for primary source classification in single frame " 

264 "processing. True if sources are not sky sources and not a parent.") 

265 ) 

266 doApCorr = pexConfig.Field( 

267 dtype=bool, 

268 default=True, 

269 doc="Run subtask to apply aperture correction" 

270 ) 

271 applyApCorr = pexConfig.ConfigurableField( 

272 target=ApplyApCorrTask, 

273 doc="Subtask to apply aperture corrections" 

274 ) 

275 # If doApCorr is False, and the exposure does not have apcorrections 

276 # already applied, the active plugins in catalogCalculation almost 

277 # certainly should not contain the characterization plugin 

278 catalogCalculation = pexConfig.ConfigurableField( 

279 target=CatalogCalculationTask, 

280 doc="Subtask to run catalogCalculation plugins on catalog" 

281 ) 

282 doInsertFakes = pexConfig.Field( 

283 dtype=bool, 

284 default=False, 

285 doc="Run fake sources injection task" 

286 ) 

287 insertFakes = pexConfig.ConfigurableField( 

288 target=BaseFakeSourcesTask, 

289 doc="Injection of fake sources for testing purposes (must be " 

290 "retargeted)" 

291 ) 

292 doComputeSummaryStats = pexConfig.Field( 

293 dtype=bool, 

294 default=True, 

295 doc="Run subtask to measure exposure summary statistics?" 

296 ) 

297 computeSummaryStats = pexConfig.ConfigurableField( 

298 target=ComputeExposureSummaryStatsTask, 

299 doc="Subtask to run computeSummaryStats on exposure" 

300 ) 

301 doWriteExposure = pexConfig.Field( 

302 dtype=bool, 

303 default=True, 

304 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

305 "normal calexp but as a fakes_calexp." 

306 ) 

307 

308 def setDefaults(self): 

309 super().setDefaults() 

310 self.detection.doTempLocalBackground = False 

311 self.deblend.maxFootprintSize = 2000 

312 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"] 

313 

314 def validate(self): 

315 super().validate() 

316 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

317 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

318 raise ValueError( 

319 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

320 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

321 ) 

322 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

323 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

324 raise ValueError( 

325 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

326 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

327 ) 

328 

329 

330## \addtogroup LSST_task_documentation 

331## \{ 

332## \page CalibrateTask 

333## \ref CalibrateTask_ "CalibrateTask" 

334## \copybrief CalibrateTask 

335## \} 

336 

337class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

338 r"""!Calibrate an exposure: measure sources and perform astrometric and 

339 photometric calibration 

340 

341 @anchor CalibrateTask_ 

342 

343 @section pipe_tasks_calibrate_Contents Contents 

344 

345 - @ref pipe_tasks_calibrate_Purpose 

346 - @ref pipe_tasks_calibrate_Initialize 

347 - @ref pipe_tasks_calibrate_IO 

348 - @ref pipe_tasks_calibrate_Config 

349 - @ref pipe_tasks_calibrate_Metadata 

350 - @ref pipe_tasks_calibrate_Debug 

351 

352 

353 @section pipe_tasks_calibrate_Purpose Description 

354 

355 Given an exposure with a good PSF model and aperture correction map 

356 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

357 operations: 

358 - Run detection and measurement 

359 - Run astrometry subtask to fit an improved WCS 

360 - Run photoCal subtask to fit the exposure's photometric zero-point 

361 

362 @section pipe_tasks_calibrate_Initialize Task initialisation 

363 

364 @copydoc \_\_init\_\_ 

365 

366 @section pipe_tasks_calibrate_IO Invoking the Task 

367 

368 If you want this task to unpersist inputs or persist outputs, then call 

369 the `runDataRef` method (a wrapper around the `run` method). 

370 

371 If you already have the inputs unpersisted and do not want to persist the 

372 output then it is more direct to call the `run` method: 

373 

374 @section pipe_tasks_calibrate_Config Configuration parameters 

375 

376 See @ref CalibrateConfig 

377 

378 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

379 

380 Exposure metadata 

381 <dl> 

382 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

383 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

384 task 

385 <dt>COLORTERM1 <dd>?? (always 0.0) 

386 <dt>COLORTERM2 <dd>?? (always 0.0) 

387 <dt>COLORTERM3 <dd>?? (always 0.0) 

388 </dl> 

389 

390 @section pipe_tasks_calibrate_Debug Debug variables 

391 

392 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

393 interface supports a flag 

394 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

395 for more about `debug.py`. 

396 

397 CalibrateTask has a debug dictionary containing one key: 

398 <dl> 

399 <dt>calibrate 

400 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

401 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

402 the meaning of the various symbols. 

403 </dl> 

404 

405 For example, put something like: 

406 @code{.py} 

407 import lsstDebug 

408 def DebugInfo(name): 

409 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

410 # call us recursively 

411 if name == "lsst.pipe.tasks.calibrate": 

412 di.display = dict( 

413 calibrate = 1, 

414 ) 

415 

416 return di 

417 

418 lsstDebug.Info = DebugInfo 

419 @endcode 

420 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

421 flag. 

422 

423 Some subtasks may have their own debug variables; see individual Task 

424 documentation. 

425 """ 

426 

427 # Example description used to live here, removed 2-20-2017 as per 

428 # https://jira.lsstcorp.org/browse/DM-9520 

429 

430 ConfigClass = CalibrateConfig 

431 _DefaultName = "calibrate" 

432 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

433 

434 def __init__(self, butler=None, astromRefObjLoader=None, 

435 photoRefObjLoader=None, icSourceSchema=None, 

436 initInputs=None, **kwargs): 

437 """!Construct a CalibrateTask 

438 

439 @param[in] butler The butler is passed to the refObjLoader constructor 

440 in case it is needed. Ignored if the refObjLoader argument 

441 provides a loader directly. 

442 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

443 that supplies an external reference catalog for astrometric 

444 calibration. May be None if the desired loader can be constructed 

445 from the butler argument or all steps requiring a reference catalog 

446 are disabled. 

447 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

448 that supplies an external reference catalog for photometric 

449 calibration. May be None if the desired loader can be constructed 

450 from the butler argument or all steps requiring a reference catalog 

451 are disabled. 

452 @param[in] icSourceSchema schema for icSource catalog, or None. 

453 Schema values specified in config.icSourceFieldsToCopy will be 

454 taken from this schema. If set to None, no values will be 

455 propagated from the icSourceCatalog 

456 @param[in,out] kwargs other keyword arguments for 

457 lsst.pipe.base.CmdLineTask 

458 """ 

459 super().__init__(**kwargs) 

460 

461 if icSourceSchema is None and butler is not None: 

462 # Use butler to read icSourceSchema from disk. 

463 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

464 

465 if icSourceSchema is None and butler is None and initInputs is not None: 

466 icSourceSchema = initInputs['icSourceSchema'].schema 

467 

468 if icSourceSchema is not None: 

469 # use a schema mapper to avoid copying each field separately 

470 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

471 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

472 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

473 

474 # Add fields to copy from an icSource catalog 

475 # and a field to indicate that the source matched a source in that 

476 # catalog. If any fields are missing then raise an exception, but 

477 # first find all missing fields in order to make the error message 

478 # more useful. 

479 self.calibSourceKey = self.schemaMapper.addOutputField( 

480 afwTable.Field["Flag"]("calib_detected", 

481 "Source was detected as an icSource")) 

482 missingFieldNames = [] 

483 for fieldName in self.config.icSourceFieldsToCopy: 

484 try: 

485 schemaItem = icSourceSchema.find(fieldName) 

486 except Exception: 

487 missingFieldNames.append(fieldName) 

488 else: 

489 # field found; if addMapping fails then raise an exception 

490 self.schemaMapper.addMapping(schemaItem.getKey()) 

491 

492 if missingFieldNames: 

493 raise RuntimeError("isSourceCat is missing fields {} " 

494 "specified in icSourceFieldsToCopy" 

495 .format(missingFieldNames)) 

496 

497 # produce a temporary schema to pass to the subtasks; finalize it 

498 # later 

499 self.schema = self.schemaMapper.editOutputSchema() 

500 else: 

501 self.schemaMapper = None 

502 self.schema = afwTable.SourceTable.makeMinimalSchema() 

503 self.makeSubtask('detection', schema=self.schema) 

504 

505 self.algMetadata = dafBase.PropertyList() 

506 

507 # Only create a subtask for fakes if configuration option is set 

508 # N.B. the config for fake object task must be retargeted to a child 

509 # of BaseFakeSourcesTask 

510 if self.config.doInsertFakes: 

511 self.makeSubtask("insertFakes") 

512 

513 if self.config.doDeblend: 

514 self.makeSubtask("deblend", schema=self.schema) 

515 if self.config.doSkySources: 

516 self.makeSubtask("skySources") 

517 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

518 self.makeSubtask('measurement', schema=self.schema, 

519 algMetadata=self.algMetadata) 

520 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

521 if self.config.doApCorr: 

522 self.makeSubtask('applyApCorr', schema=self.schema) 

523 self.makeSubtask('catalogCalculation', schema=self.schema) 

524 

525 if self.config.doAstrometry: 

526 if astromRefObjLoader is None and butler is not None: 

527 self.makeSubtask('astromRefObjLoader', butler=butler) 

528 astromRefObjLoader = self.astromRefObjLoader 

529 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

530 schema=self.schema) 

531 if self.config.doPhotoCal: 

532 if photoRefObjLoader is None and butler is not None: 

533 self.makeSubtask('photoRefObjLoader', butler=butler) 

534 photoRefObjLoader = self.photoRefObjLoader 

535 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

536 schema=self.schema) 

537 if self.config.doComputeSummaryStats: 

538 self.makeSubtask('computeSummaryStats') 

539 

540 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

541 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

542 "reference object loaders.") 

543 

544 if self.schemaMapper is not None: 

545 # finalize the schema 

546 self.schema = self.schemaMapper.getOutputSchema() 

547 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

548 

549 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

550 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

551 self.outputSchema = sourceCatSchema 

552 

553 @timeMethod 

554 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

555 doUnpersist=True): 

556 """!Calibrate an exposure, optionally unpersisting inputs and 

557 persisting outputs. 

558 

559 This is a wrapper around the `run` method that unpersists inputs 

560 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

561 

562 @param[in] dataRef butler data reference corresponding to a science 

563 image 

564 @param[in,out] exposure characterized exposure (an 

565 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

566 icExp and icBackground. See `run` method for details of what is 

567 read and written. 

568 @param[in,out] background initial model of background already 

569 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

570 None if no background has been subtracted, though that is unusual 

571 for calibration. A refined background model is output. Ignored if 

572 exposure is None. 

573 @param[in] icSourceCat catalog from which to copy the fields specified 

574 by icSourceKeys, or None; 

575 @param[in] doUnpersist unpersist data: 

576 - if True, exposure, background and icSourceCat are read from 

577 dataRef and those three arguments must all be None; 

578 - if False the exposure must be provided; background and 

579 icSourceCat are optional. True is intended for running as a 

580 command-line task, False for running as a subtask 

581 @return same data as the calibrate method 

582 """ 

583 self.log.info("Processing %s", dataRef.dataId) 

584 

585 if doUnpersist: 

586 if any(item is not None for item in (exposure, background, 

587 icSourceCat)): 

588 raise RuntimeError("doUnpersist true; exposure, background " 

589 "and icSourceCat must all be None") 

590 exposure = dataRef.get("icExp", immediate=True) 

591 background = dataRef.get("icExpBackground", immediate=True) 

592 icSourceCat = dataRef.get("icSrc", immediate=True) 

593 elif exposure is None: 

594 raise RuntimeError("doUnpersist false; exposure must be provided") 

595 

596 exposureIdInfo = dataRef.get("expIdInfo") 

597 

598 calRes = self.run( 

599 exposure=exposure, 

600 exposureIdInfo=exposureIdInfo, 

601 background=background, 

602 icSourceCat=icSourceCat, 

603 ) 

604 

605 if self.config.doWrite: 

606 self.writeOutputs( 

607 dataRef=dataRef, 

608 exposure=calRes.exposure, 

609 background=calRes.background, 

610 sourceCat=calRes.sourceCat, 

611 astromMatches=calRes.astromMatches, 

612 matchMeta=calRes.matchMeta, 

613 ) 

614 

615 return calRes 

616 

617 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

618 inputs = butlerQC.get(inputRefs) 

619 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

620 

621 if self.config.doAstrometry: 

622 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

623 for ref in inputRefs.astromRefCat], 

624 refCats=inputs.pop('astromRefCat'), 

625 config=self.config.astromRefObjLoader, log=self.log) 

626 self.astrometry.setRefObjLoader(refObjLoader) 

627 

628 if self.config.doPhotoCal: 

629 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

630 for ref in inputRefs.photoRefCat], 

631 refCats=inputs.pop('photoRefCat'), 

632 config=self.config.photoRefObjLoader, 

633 log=self.log) 

634 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

635 

636 outputs = self.run(**inputs) 

637 

638 if self.config.doWriteMatches and self.config.doAstrometry: 

639 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

640 normalizedMatches.table.setMetadata(outputs.matchMeta) 

641 if self.config.doWriteMatchesDenormalized: 

642 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

643 outputs.matchesDenormalized = denormMatches 

644 outputs.matches = normalizedMatches 

645 butlerQC.put(outputs, outputRefs) 

646 

647 @timeMethod 

648 def run(self, exposure, exposureIdInfo=None, background=None, 

649 icSourceCat=None): 

650 """!Calibrate an exposure (science image or coadd) 

651 

652 @param[in,out] exposure exposure to calibrate (an 

653 lsst.afw.image.ExposureF or similar); 

654 in: 

655 - MaskedImage 

656 - Psf 

657 out: 

658 - MaskedImage has background subtracted 

659 - Wcs is replaced 

660 - PhotoCalib is replaced 

661 @param[in] exposureIdInfo ID info for exposure (an 

662 lsst.obs.base.ExposureIdInfo) If not provided, returned 

663 SourceCatalog IDs will not be globally unique. 

664 @param[in,out] background background model already subtracted from 

665 exposure (an lsst.afw.math.BackgroundList). May be None if no 

666 background has been subtracted, though that is unusual for 

667 calibration. A refined background model is output. 

668 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

669 from which we can copy some fields. 

670 

671 @return pipe_base Struct containing these fields: 

672 - exposure calibrate science exposure with refined WCS and PhotoCalib 

673 - background model of background subtracted from exposure (an 

674 lsst.afw.math.BackgroundList) 

675 - sourceCat catalog of measured sources 

676 - astromMatches list of source/refObj matches from the astrometry 

677 solver 

678 """ 

679 # detect, deblend and measure sources 

680 if exposureIdInfo is None: 

681 exposureIdInfo = ExposureIdInfo() 

682 

683 if background is None: 

684 background = BackgroundList() 

685 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

686 table = SourceTable.make(self.schema, sourceIdFactory) 

687 table.setMetadata(self.algMetadata) 

688 

689 detRes = self.detection.run(table=table, exposure=exposure, 

690 doSmooth=True) 

691 sourceCat = detRes.sources 

692 if detRes.fpSets.background: 

693 for bg in detRes.fpSets.background: 

694 background.append(bg) 

695 if self.config.doSkySources: 

696 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

697 if skySourceFootprints: 

698 for foot in skySourceFootprints: 

699 s = sourceCat.addNew() 

700 s.setFootprint(foot) 

701 s.set(self.skySourceKey, True) 

702 if self.config.doDeblend: 

703 self.deblend.run(exposure=exposure, sources=sourceCat) 

704 self.measurement.run( 

705 measCat=sourceCat, 

706 exposure=exposure, 

707 exposureId=exposureIdInfo.expId 

708 ) 

709 if self.config.doApCorr: 

710 self.applyApCorr.run( 

711 catalog=sourceCat, 

712 apCorrMap=exposure.getInfo().getApCorrMap() 

713 ) 

714 self.catalogCalculation.run(sourceCat) 

715 

716 self.setPrimaryFlags.run(sourceCat) 

717 

718 if icSourceCat is not None and \ 

719 len(self.config.icSourceFieldsToCopy) > 0: 

720 self.copyIcSourceFields(icSourceCat=icSourceCat, 

721 sourceCat=sourceCat) 

722 

723 # TODO DM-11568: this contiguous check-and-copy could go away if we 

724 # reserve enough space during SourceDetection and/or SourceDeblend. 

725 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

726 # contiguity now, so views are preserved from here on. 

727 if not sourceCat.isContiguous(): 

728 sourceCat = sourceCat.copy(deep=True) 

729 

730 # perform astrometry calibration: 

731 # fit an improved WCS and update the exposure's WCS in place 

732 astromMatches = None 

733 matchMeta = None 

734 if self.config.doAstrometry: 

735 try: 

736 astromRes = self.astrometry.run( 

737 exposure=exposure, 

738 sourceCat=sourceCat, 

739 ) 

740 astromMatches = astromRes.matches 

741 matchMeta = astromRes.matchMeta 

742 except Exception as e: 

743 if self.config.requireAstrometry: 

744 raise 

745 self.log.warning("Unable to perform astrometric calibration " 

746 "(%s): attempting to proceed", e) 

747 

748 # compute photometric calibration 

749 if self.config.doPhotoCal: 

750 try: 

751 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

752 exposure.setPhotoCalib(photoRes.photoCalib) 

753 # TODO: reword this to phrase it in terms of the calibration factor? 

754 self.log.info("Photometric zero-point: %f", 

755 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

756 self.setMetadata(exposure=exposure, photoRes=photoRes) 

757 except Exception as e: 

758 if self.config.requirePhotoCal: 

759 raise 

760 self.log.warning("Unable to perform photometric calibration " 

761 "(%s): attempting to proceed", e) 

762 self.setMetadata(exposure=exposure, photoRes=None) 

763 

764 if self.config.doInsertFakes: 

765 self.insertFakes.run(exposure, background=background) 

766 

767 table = SourceTable.make(self.schema, sourceIdFactory) 

768 table.setMetadata(self.algMetadata) 

769 

770 detRes = self.detection.run(table=table, exposure=exposure, 

771 doSmooth=True) 

772 sourceCat = detRes.sources 

773 if detRes.fpSets.background: 

774 for bg in detRes.fpSets.background: 

775 background.append(bg) 

776 if self.config.doDeblend: 

777 self.deblend.run(exposure=exposure, sources=sourceCat) 

778 self.measurement.run( 

779 measCat=sourceCat, 

780 exposure=exposure, 

781 exposureId=exposureIdInfo.expId 

782 ) 

783 if self.config.doApCorr: 

784 self.applyApCorr.run( 

785 catalog=sourceCat, 

786 apCorrMap=exposure.getInfo().getApCorrMap() 

787 ) 

788 self.catalogCalculation.run(sourceCat) 

789 

790 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

791 self.copyIcSourceFields(icSourceCat=icSourceCat, 

792 sourceCat=sourceCat) 

793 

794 if self.config.doComputeSummaryStats: 

795 summary = self.computeSummaryStats.run(exposure=exposure, 

796 sources=sourceCat, 

797 background=background) 

798 exposure.getInfo().setSummaryStats(summary) 

799 

800 frame = getDebugFrame(self._display, "calibrate") 

801 if frame: 

802 displayAstrometry( 

803 sourceCat=sourceCat, 

804 exposure=exposure, 

805 matches=astromMatches, 

806 frame=frame, 

807 pause=False, 

808 ) 

809 

810 return pipeBase.Struct( 

811 exposure=exposure, 

812 background=background, 

813 sourceCat=sourceCat, 

814 astromMatches=astromMatches, 

815 matchMeta=matchMeta, 

816 # These are duplicate entries with different names for use with 

817 # gen3 middleware 

818 outputExposure=exposure, 

819 outputCat=sourceCat, 

820 outputBackground=background, 

821 ) 

822 

823 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

824 astromMatches, matchMeta): 

825 """Write output data to the output repository 

826 

827 @param[in] dataRef butler data reference corresponding to a science 

828 image 

829 @param[in] exposure exposure to write 

830 @param[in] background background model for exposure 

831 @param[in] sourceCat catalog of measured sources 

832 @param[in] astromMatches list of source/refObj matches from the 

833 astrometry solver 

834 """ 

835 dataRef.put(sourceCat, "src") 

836 if self.config.doWriteMatches and astromMatches is not None: 

837 normalizedMatches = afwTable.packMatches(astromMatches) 

838 normalizedMatches.table.setMetadata(matchMeta) 

839 dataRef.put(normalizedMatches, "srcMatch") 

840 if self.config.doWriteMatchesDenormalized: 

841 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

842 dataRef.put(denormMatches, "srcMatchFull") 

843 if self.config.doWriteExposure: 

844 dataRef.put(exposure, "calexp") 

845 dataRef.put(background, "calexpBackground") 

846 

847 def getSchemaCatalogs(self): 

848 """Return a dict of empty catalogs for each catalog dataset produced 

849 by this task. 

850 """ 

851 sourceCat = afwTable.SourceCatalog(self.schema) 

852 sourceCat.getTable().setMetadata(self.algMetadata) 

853 return {"src": sourceCat} 

854 

855 def setMetadata(self, exposure, photoRes=None): 

856 """!Set task and exposure metadata 

857 

858 Logs a warning and continues if needed data is missing. 

859 

860 @param[in,out] exposure exposure whose metadata is to be set 

861 @param[in] photoRes results of running photoCal; if None then it was 

862 not run 

863 """ 

864 if photoRes is None: 

865 return 

866 

867 metadata = exposure.getMetadata() 

868 

869 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

870 try: 

871 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

872 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

873 except Exception: 

874 self.log.warning("Could not set normalized MAGZERO in header: no " 

875 "exposure time") 

876 magZero = math.nan 

877 

878 try: 

879 metadata.set('MAGZERO', magZero) 

880 metadata.set('MAGZERO_RMS', photoRes.sigma) 

881 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

882 metadata.set('COLORTERM1', 0.0) 

883 metadata.set('COLORTERM2', 0.0) 

884 metadata.set('COLORTERM3', 0.0) 

885 except Exception as e: 

886 self.log.warning("Could not set exposure metadata: %s", e) 

887 

888 def copyIcSourceFields(self, icSourceCat, sourceCat): 

889 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

890 

891 @param[in] icSourceCat catalog from which to copy fields 

892 @param[in,out] sourceCat catalog to which to copy fields 

893 

894 The fields copied are those specified by `config.icSourceFieldsToCopy` 

895 that actually exist in the schema. This was set up by the constructor 

896 using self.schemaMapper. 

897 """ 

898 if self.schemaMapper is None: 

899 raise RuntimeError("To copy icSource fields you must specify " 

900 "icSourceSchema nd icSourceKeys when " 

901 "constructing this task") 

902 if icSourceCat is None or sourceCat is None: 

903 raise RuntimeError("icSourceCat and sourceCat must both be " 

904 "specified") 

905 if len(self.config.icSourceFieldsToCopy) == 0: 

906 self.log.warning("copyIcSourceFields doing nothing because " 

907 "icSourceFieldsToCopy is empty") 

908 return 

909 

910 mc = afwTable.MatchControl() 

911 mc.findOnlyClosest = False # return all matched objects 

912 matches = afwTable.matchXy(icSourceCat, sourceCat, 

913 self.config.matchRadiusPix, mc) 

914 if self.config.doDeblend: 

915 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

916 # if deblended, keep children 

917 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

918 

919 # Because we had to allow multiple matches to handle parents, we now 

920 # need to prune to the best matches 

921 # closest matches as a dict of icSourceCat source ID: 

922 # (icSourceCat source, sourceCat source, distance in pixels) 

923 bestMatches = {} 

924 for m0, m1, d in matches: 

925 id0 = m0.getId() 

926 match = bestMatches.get(id0) 

927 if match is None or d <= match[2]: 

928 bestMatches[id0] = (m0, m1, d) 

929 matches = list(bestMatches.values()) 

930 

931 # Check that no sourceCat sources are listed twice (we already know 

932 # that each match has a unique icSourceCat source ID, due to using 

933 # that ID as the key in bestMatches) 

934 numMatches = len(matches) 

935 numUniqueSources = len(set(m[1].getId() for m in matches)) 

936 if numUniqueSources != numMatches: 

937 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

938 "sources", numMatches, numUniqueSources) 

939 

940 self.log.info("Copying flags from icSourceCat to sourceCat for " 

941 "%d sources", numMatches) 

942 

943 # For each match: set the calibSourceKey flag and copy the desired 

944 # fields 

945 for icSrc, src, d in matches: 

946 src.setFlag(self.calibSourceKey, True) 

947 # src.assign copies the footprint from icSrc, which we don't want 

948 # (DM-407) 

949 # so set icSrc's footprint to src's footprint before src.assign, 

950 # then restore it 

951 icSrcFootprint = icSrc.getFootprint() 

952 try: 

953 icSrc.setFootprint(src.getFootprint()) 

954 src.assign(icSrc, self.schemaMapper) 

955 finally: 

956 icSrc.setFootprint(icSrcFootprint)