Coverage for python/lsst/pipe/tasks/calibrate.py: 18%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

336 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from lsst.utils.timer import timeMethod 

41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

42from .fakes import BaseFakeSourcesTask 

43from .photoCal import PhotoCalTask 

44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

45 

46 

47__all__ = ["CalibrateConfig", "CalibrateTask"] 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="cal_ref_cat", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="cal_ref_cat", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask""" 

155 doWrite = pexConfig.Field( 

156 dtype=bool, 

157 default=True, 

158 doc="Save calibration results?", 

159 ) 

160 doWriteHeavyFootprintsInSources = pexConfig.Field( 

161 dtype=bool, 

162 default=True, 

163 doc="Include HeavyFootprint data in source table? If false then heavy " 

164 "footprints are saved as normal footprints, which saves some space" 

165 ) 

166 doWriteMatches = pexConfig.Field( 

167 dtype=bool, 

168 default=True, 

169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

170 ) 

171 doWriteMatchesDenormalized = pexConfig.Field( 

172 dtype=bool, 

173 default=False, 

174 doc=("Write reference matches in denormalized format? " 

175 "This format uses more disk space, but is more convenient to " 

176 "read. Ignored if doWriteMatches=False or doWrite=False."), 

177 ) 

178 doAstrometry = pexConfig.Field( 

179 dtype=bool, 

180 default=True, 

181 doc="Perform astrometric calibration?", 

182 ) 

183 astromRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for astrometric calibration", 

186 ) 

187 photoRefObjLoader = pexConfig.ConfigurableField( 

188 target=LoadIndexedReferenceObjectsTask, 

189 doc="reference object loader for photometric calibration", 

190 ) 

191 astrometry = pexConfig.ConfigurableField( 

192 target=AstrometryTask, 

193 doc="Perform astrometric calibration to refine the WCS", 

194 ) 

195 requireAstrometry = pexConfig.Field( 

196 dtype=bool, 

197 default=True, 

198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

199 "false."), 

200 ) 

201 doPhotoCal = pexConfig.Field( 

202 dtype=bool, 

203 default=True, 

204 doc="Perform phometric calibration?", 

205 ) 

206 requirePhotoCal = pexConfig.Field( 

207 dtype=bool, 

208 default=True, 

209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

210 "false."), 

211 ) 

212 photoCal = pexConfig.ConfigurableField( 

213 target=PhotoCalTask, 

214 doc="Perform photometric calibration", 

215 ) 

216 icSourceFieldsToCopy = pexConfig.ListField( 

217 dtype=str, 

218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

219 doc=("Fields to copy from the icSource catalog to the output catalog " 

220 "for matching sources Any missing fields will trigger a " 

221 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

222 ) 

223 matchRadiusPix = pexConfig.Field( 

224 dtype=float, 

225 default=3, 

226 doc=("Match radius for matching icSourceCat objects to sourceCat " 

227 "objects (pixels)"), 

228 ) 

229 checkUnitsParseStrict = pexConfig.Field( 

230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

231 "'warn' or 'silent'"), 

232 dtype=str, 

233 default="raise", 

234 ) 

235 detection = pexConfig.ConfigurableField( 

236 target=SourceDetectionTask, 

237 doc="Detect sources" 

238 ) 

239 doDeblend = pexConfig.Field( 

240 dtype=bool, 

241 default=True, 

242 doc="Run deblender input exposure" 

243 ) 

244 deblend = pexConfig.ConfigurableField( 

245 target=SourceDeblendTask, 

246 doc="Split blended sources into their components" 

247 ) 

248 doSkySources = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Generate sky sources?", 

252 ) 

253 skySources = pexConfig.ConfigurableField( 

254 target=SkyObjectsTask, 

255 doc="Generate sky sources", 

256 ) 

257 measurement = pexConfig.ConfigurableField( 

258 target=SingleFrameMeasurementTask, 

259 doc="Measure sources" 

260 ) 

261 postCalibrationMeasurement = pexConfig.ConfigurableField( 

262 target=SingleFrameMeasurementTask, 

263 doc="Second round of measurement for plugins that need to be run after photocal" 

264 ) 

265 setPrimaryFlags = pexConfig.ConfigurableField( 

266 target=SetPrimaryFlagsTask, 

267 doc=("Set flags for primary source classification in single frame " 

268 "processing. True if sources are not sky sources and not a parent.") 

269 ) 

270 doApCorr = pexConfig.Field( 

271 dtype=bool, 

272 default=True, 

273 doc="Run subtask to apply aperture correction" 

274 ) 

275 applyApCorr = pexConfig.ConfigurableField( 

276 target=ApplyApCorrTask, 

277 doc="Subtask to apply aperture corrections" 

278 ) 

279 # If doApCorr is False, and the exposure does not have apcorrections 

280 # already applied, the active plugins in catalogCalculation almost 

281 # certainly should not contain the characterization plugin 

282 catalogCalculation = pexConfig.ConfigurableField( 

283 target=CatalogCalculationTask, 

284 doc="Subtask to run catalogCalculation plugins on catalog" 

285 ) 

286 doInsertFakes = pexConfig.Field( 

287 dtype=bool, 

288 default=False, 

289 doc="Run fake sources injection task" 

290 ) 

291 insertFakes = pexConfig.ConfigurableField( 

292 target=BaseFakeSourcesTask, 

293 doc="Injection of fake sources for testing purposes (must be " 

294 "retargeted)" 

295 ) 

296 doComputeSummaryStats = pexConfig.Field( 

297 dtype=bool, 

298 default=True, 

299 doc="Run subtask to measure exposure summary statistics?" 

300 ) 

301 computeSummaryStats = pexConfig.ConfigurableField( 

302 target=ComputeExposureSummaryStatsTask, 

303 doc="Subtask to run computeSummaryStats on exposure" 

304 ) 

305 doWriteExposure = pexConfig.Field( 

306 dtype=bool, 

307 default=True, 

308 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

309 "normal calexp but as a fakes_calexp." 

310 ) 

311 

312 def setDefaults(self): 

313 super().setDefaults() 

314 self.detection.doTempLocalBackground = False 

315 self.deblend.maxFootprintSize = 2000 

316 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

317 self.postCalibrationMeasurement.doReplaceWithNoise = False 

318 for key in self.postCalibrationMeasurement.slots: 

319 setattr(self.postCalibrationMeasurement.slots, key, None) 

320 

321 def validate(self): 

322 super().validate() 

323 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

324 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

325 raise ValueError( 

326 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

327 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

328 ) 

329 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

330 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

331 raise ValueError( 

332 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

333 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

334 ) 

335 

336 

337## \addtogroup LSST_task_documentation 

338## \{ 

339## \page CalibrateTask 

340## \ref CalibrateTask_ "CalibrateTask" 

341## \copybrief CalibrateTask 

342## \} 

343 

344class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

345 r"""!Calibrate an exposure: measure sources and perform astrometric and 

346 photometric calibration 

347 

348 @anchor CalibrateTask_ 

349 

350 @section pipe_tasks_calibrate_Contents Contents 

351 

352 - @ref pipe_tasks_calibrate_Purpose 

353 - @ref pipe_tasks_calibrate_Initialize 

354 - @ref pipe_tasks_calibrate_IO 

355 - @ref pipe_tasks_calibrate_Config 

356 - @ref pipe_tasks_calibrate_Metadata 

357 - @ref pipe_tasks_calibrate_Debug 

358 

359 

360 @section pipe_tasks_calibrate_Purpose Description 

361 

362 Given an exposure with a good PSF model and aperture correction map 

363 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

364 operations: 

365 - Run detection and measurement 

366 - Run astrometry subtask to fit an improved WCS 

367 - Run photoCal subtask to fit the exposure's photometric zero-point 

368 

369 @section pipe_tasks_calibrate_Initialize Task initialisation 

370 

371 @copydoc \_\_init\_\_ 

372 

373 @section pipe_tasks_calibrate_IO Invoking the Task 

374 

375 If you want this task to unpersist inputs or persist outputs, then call 

376 the `runDataRef` method (a wrapper around the `run` method). 

377 

378 If you already have the inputs unpersisted and do not want to persist the 

379 output then it is more direct to call the `run` method: 

380 

381 @section pipe_tasks_calibrate_Config Configuration parameters 

382 

383 See @ref CalibrateConfig 

384 

385 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

386 

387 Exposure metadata 

388 <dl> 

389 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

390 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

391 task 

392 <dt>COLORTERM1 <dd>?? (always 0.0) 

393 <dt>COLORTERM2 <dd>?? (always 0.0) 

394 <dt>COLORTERM3 <dd>?? (always 0.0) 

395 </dl> 

396 

397 @section pipe_tasks_calibrate_Debug Debug variables 

398 

399 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

400 interface supports a flag 

401 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

402 for more about `debug.py`. 

403 

404 CalibrateTask has a debug dictionary containing one key: 

405 <dl> 

406 <dt>calibrate 

407 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

408 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

409 the meaning of the various symbols. 

410 </dl> 

411 

412 For example, put something like: 

413 @code{.py} 

414 import lsstDebug 

415 def DebugInfo(name): 

416 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

417 # call us recursively 

418 if name == "lsst.pipe.tasks.calibrate": 

419 di.display = dict( 

420 calibrate = 1, 

421 ) 

422 

423 return di 

424 

425 lsstDebug.Info = DebugInfo 

426 @endcode 

427 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

428 flag. 

429 

430 Some subtasks may have their own debug variables; see individual Task 

431 documentation. 

432 """ 

433 

434 # Example description used to live here, removed 2-20-2017 as per 

435 # https://jira.lsstcorp.org/browse/DM-9520 

436 

437 ConfigClass = CalibrateConfig 

438 _DefaultName = "calibrate" 

439 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

440 

441 def __init__(self, butler=None, astromRefObjLoader=None, 

442 photoRefObjLoader=None, icSourceSchema=None, 

443 initInputs=None, **kwargs): 

444 """!Construct a CalibrateTask 

445 

446 @param[in] butler The butler is passed to the refObjLoader constructor 

447 in case it is needed. Ignored if the refObjLoader argument 

448 provides a loader directly. 

449 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

450 that supplies an external reference catalog for astrometric 

451 calibration. May be None if the desired loader can be constructed 

452 from the butler argument or all steps requiring a reference catalog 

453 are disabled. 

454 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

455 that supplies an external reference catalog for photometric 

456 calibration. May be None if the desired loader can be constructed 

457 from the butler argument or all steps requiring a reference catalog 

458 are disabled. 

459 @param[in] icSourceSchema schema for icSource catalog, or None. 

460 Schema values specified in config.icSourceFieldsToCopy will be 

461 taken from this schema. If set to None, no values will be 

462 propagated from the icSourceCatalog 

463 @param[in,out] kwargs other keyword arguments for 

464 lsst.pipe.base.CmdLineTask 

465 """ 

466 super().__init__(**kwargs) 

467 

468 if icSourceSchema is None and butler is not None: 

469 # Use butler to read icSourceSchema from disk. 

470 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

471 

472 if icSourceSchema is None and butler is None and initInputs is not None: 

473 icSourceSchema = initInputs['icSourceSchema'].schema 

474 

475 if icSourceSchema is not None: 

476 # use a schema mapper to avoid copying each field separately 

477 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

478 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

479 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

480 

481 # Add fields to copy from an icSource catalog 

482 # and a field to indicate that the source matched a source in that 

483 # catalog. If any fields are missing then raise an exception, but 

484 # first find all missing fields in order to make the error message 

485 # more useful. 

486 self.calibSourceKey = self.schemaMapper.addOutputField( 

487 afwTable.Field["Flag"]("calib_detected", 

488 "Source was detected as an icSource")) 

489 missingFieldNames = [] 

490 for fieldName in self.config.icSourceFieldsToCopy: 

491 try: 

492 schemaItem = icSourceSchema.find(fieldName) 

493 except Exception: 

494 missingFieldNames.append(fieldName) 

495 else: 

496 # field found; if addMapping fails then raise an exception 

497 self.schemaMapper.addMapping(schemaItem.getKey()) 

498 

499 if missingFieldNames: 

500 raise RuntimeError("isSourceCat is missing fields {} " 

501 "specified in icSourceFieldsToCopy" 

502 .format(missingFieldNames)) 

503 

504 # produce a temporary schema to pass to the subtasks; finalize it 

505 # later 

506 self.schema = self.schemaMapper.editOutputSchema() 

507 else: 

508 self.schemaMapper = None 

509 self.schema = afwTable.SourceTable.makeMinimalSchema() 

510 self.makeSubtask('detection', schema=self.schema) 

511 

512 self.algMetadata = dafBase.PropertyList() 

513 

514 # Only create a subtask for fakes if configuration option is set 

515 # N.B. the config for fake object task must be retargeted to a child 

516 # of BaseFakeSourcesTask 

517 if self.config.doInsertFakes: 

518 self.makeSubtask("insertFakes") 

519 

520 if self.config.doDeblend: 

521 self.makeSubtask("deblend", schema=self.schema) 

522 if self.config.doSkySources: 

523 self.makeSubtask("skySources") 

524 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

525 self.makeSubtask('measurement', schema=self.schema, 

526 algMetadata=self.algMetadata) 

527 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

528 algMetadata=self.algMetadata) 

529 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

530 if self.config.doApCorr: 

531 self.makeSubtask('applyApCorr', schema=self.schema) 

532 self.makeSubtask('catalogCalculation', schema=self.schema) 

533 

534 if self.config.doAstrometry: 

535 if astromRefObjLoader is None and butler is not None: 

536 self.makeSubtask('astromRefObjLoader', butler=butler) 

537 astromRefObjLoader = self.astromRefObjLoader 

538 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

539 schema=self.schema) 

540 if self.config.doPhotoCal: 

541 if photoRefObjLoader is None and butler is not None: 

542 self.makeSubtask('photoRefObjLoader', butler=butler) 

543 photoRefObjLoader = self.photoRefObjLoader 

544 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

545 schema=self.schema) 

546 if self.config.doComputeSummaryStats: 

547 self.makeSubtask('computeSummaryStats') 

548 

549 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

550 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

551 "reference object loaders.") 

552 

553 if self.schemaMapper is not None: 

554 # finalize the schema 

555 self.schema = self.schemaMapper.getOutputSchema() 

556 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

557 

558 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

559 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

560 self.outputSchema = sourceCatSchema 

561 

562 @timeMethod 

563 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

564 doUnpersist=True): 

565 """!Calibrate an exposure, optionally unpersisting inputs and 

566 persisting outputs. 

567 

568 This is a wrapper around the `run` method that unpersists inputs 

569 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

570 

571 @param[in] dataRef butler data reference corresponding to a science 

572 image 

573 @param[in,out] exposure characterized exposure (an 

574 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

575 icExp and icBackground. See `run` method for details of what is 

576 read and written. 

577 @param[in,out] background initial model of background already 

578 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

579 None if no background has been subtracted, though that is unusual 

580 for calibration. A refined background model is output. Ignored if 

581 exposure is None. 

582 @param[in] icSourceCat catalog from which to copy the fields specified 

583 by icSourceKeys, or None; 

584 @param[in] doUnpersist unpersist data: 

585 - if True, exposure, background and icSourceCat are read from 

586 dataRef and those three arguments must all be None; 

587 - if False the exposure must be provided; background and 

588 icSourceCat are optional. True is intended for running as a 

589 command-line task, False for running as a subtask 

590 @return same data as the calibrate method 

591 """ 

592 self.log.info("Processing %s", dataRef.dataId) 

593 

594 if doUnpersist: 

595 if any(item is not None for item in (exposure, background, 

596 icSourceCat)): 

597 raise RuntimeError("doUnpersist true; exposure, background " 

598 "and icSourceCat must all be None") 

599 exposure = dataRef.get("icExp", immediate=True) 

600 background = dataRef.get("icExpBackground", immediate=True) 

601 icSourceCat = dataRef.get("icSrc", immediate=True) 

602 elif exposure is None: 

603 raise RuntimeError("doUnpersist false; exposure must be provided") 

604 

605 exposureIdInfo = dataRef.get("expIdInfo") 

606 

607 calRes = self.run( 

608 exposure=exposure, 

609 exposureIdInfo=exposureIdInfo, 

610 background=background, 

611 icSourceCat=icSourceCat, 

612 ) 

613 

614 if self.config.doWrite: 

615 self.writeOutputs( 

616 dataRef=dataRef, 

617 exposure=calRes.exposure, 

618 background=calRes.background, 

619 sourceCat=calRes.sourceCat, 

620 astromMatches=calRes.astromMatches, 

621 matchMeta=calRes.matchMeta, 

622 ) 

623 

624 return calRes 

625 

626 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

627 inputs = butlerQC.get(inputRefs) 

628 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

629 

630 if self.config.doAstrometry: 

631 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

632 for ref in inputRefs.astromRefCat], 

633 refCats=inputs.pop('astromRefCat'), 

634 config=self.config.astromRefObjLoader, log=self.log) 

635 self.astrometry.setRefObjLoader(refObjLoader) 

636 

637 if self.config.doPhotoCal: 

638 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

639 for ref in inputRefs.photoRefCat], 

640 refCats=inputs.pop('photoRefCat'), 

641 config=self.config.photoRefObjLoader, 

642 log=self.log) 

643 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

644 

645 outputs = self.run(**inputs) 

646 

647 if self.config.doWriteMatches and self.config.doAstrometry: 

648 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

649 normalizedMatches.table.setMetadata(outputs.matchMeta) 

650 if self.config.doWriteMatchesDenormalized: 

651 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

652 outputs.matchesDenormalized = denormMatches 

653 outputs.matches = normalizedMatches 

654 butlerQC.put(outputs, outputRefs) 

655 

656 @timeMethod 

657 def run(self, exposure, exposureIdInfo=None, background=None, 

658 icSourceCat=None): 

659 """!Calibrate an exposure (science image or coadd) 

660 

661 @param[in,out] exposure exposure to calibrate (an 

662 lsst.afw.image.ExposureF or similar); 

663 in: 

664 - MaskedImage 

665 - Psf 

666 out: 

667 - MaskedImage has background subtracted 

668 - Wcs is replaced 

669 - PhotoCalib is replaced 

670 @param[in] exposureIdInfo ID info for exposure (an 

671 lsst.obs.base.ExposureIdInfo) If not provided, returned 

672 SourceCatalog IDs will not be globally unique. 

673 @param[in,out] background background model already subtracted from 

674 exposure (an lsst.afw.math.BackgroundList). May be None if no 

675 background has been subtracted, though that is unusual for 

676 calibration. A refined background model is output. 

677 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

678 from which we can copy some fields. 

679 

680 @return pipe_base Struct containing these fields: 

681 - exposure calibrate science exposure with refined WCS and PhotoCalib 

682 - background model of background subtracted from exposure (an 

683 lsst.afw.math.BackgroundList) 

684 - sourceCat catalog of measured sources 

685 - astromMatches list of source/refObj matches from the astrometry 

686 solver 

687 """ 

688 # detect, deblend and measure sources 

689 if exposureIdInfo is None: 

690 exposureIdInfo = ExposureIdInfo() 

691 

692 if background is None: 

693 background = BackgroundList() 

694 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

695 table = SourceTable.make(self.schema, sourceIdFactory) 

696 table.setMetadata(self.algMetadata) 

697 

698 detRes = self.detection.run(table=table, exposure=exposure, 

699 doSmooth=True) 

700 sourceCat = detRes.sources 

701 if detRes.fpSets.background: 

702 for bg in detRes.fpSets.background: 

703 background.append(bg) 

704 if self.config.doSkySources: 

705 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

706 if skySourceFootprints: 

707 for foot in skySourceFootprints: 

708 s = sourceCat.addNew() 

709 s.setFootprint(foot) 

710 s.set(self.skySourceKey, True) 

711 if self.config.doDeblend: 

712 self.deblend.run(exposure=exposure, sources=sourceCat) 

713 self.measurement.run( 

714 measCat=sourceCat, 

715 exposure=exposure, 

716 exposureId=exposureIdInfo.expId 

717 ) 

718 if self.config.doApCorr: 

719 self.applyApCorr.run( 

720 catalog=sourceCat, 

721 apCorrMap=exposure.getInfo().getApCorrMap() 

722 ) 

723 self.catalogCalculation.run(sourceCat) 

724 

725 self.setPrimaryFlags.run(sourceCat) 

726 

727 if icSourceCat is not None and \ 

728 len(self.config.icSourceFieldsToCopy) > 0: 

729 self.copyIcSourceFields(icSourceCat=icSourceCat, 

730 sourceCat=sourceCat) 

731 

732 # TODO DM-11568: this contiguous check-and-copy could go away if we 

733 # reserve enough space during SourceDetection and/or SourceDeblend. 

734 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

735 # contiguity now, so views are preserved from here on. 

736 if not sourceCat.isContiguous(): 

737 sourceCat = sourceCat.copy(deep=True) 

738 

739 # perform astrometry calibration: 

740 # fit an improved WCS and update the exposure's WCS in place 

741 astromMatches = None 

742 matchMeta = None 

743 if self.config.doAstrometry: 

744 try: 

745 astromRes = self.astrometry.run( 

746 exposure=exposure, 

747 sourceCat=sourceCat, 

748 ) 

749 astromMatches = astromRes.matches 

750 matchMeta = astromRes.matchMeta 

751 except Exception as e: 

752 if self.config.requireAstrometry: 

753 raise 

754 self.log.warning("Unable to perform astrometric calibration " 

755 "(%s): attempting to proceed", e) 

756 

757 # compute photometric calibration 

758 if self.config.doPhotoCal: 

759 try: 

760 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

761 exposure.setPhotoCalib(photoRes.photoCalib) 

762 # TODO: reword this to phrase it in terms of the calibration factor? 

763 self.log.info("Photometric zero-point: %f", 

764 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

765 self.setMetadata(exposure=exposure, photoRes=photoRes) 

766 except Exception as e: 

767 if self.config.requirePhotoCal: 

768 raise 

769 self.log.warning("Unable to perform photometric calibration " 

770 "(%s): attempting to proceed", e) 

771 self.setMetadata(exposure=exposure, photoRes=None) 

772 

773 self.postCalibrationMeasurement.run( 

774 measCat=sourceCat, 

775 exposure=exposure, 

776 exposureId=exposureIdInfo.expId 

777 ) 

778 

779 if self.config.doInsertFakes: 

780 self.insertFakes.run(exposure, background=background) 

781 

782 table = SourceTable.make(self.schema, sourceIdFactory) 

783 table.setMetadata(self.algMetadata) 

784 

785 detRes = self.detection.run(table=table, exposure=exposure, 

786 doSmooth=True) 

787 sourceCat = detRes.sources 

788 if detRes.fpSets.background: 

789 for bg in detRes.fpSets.background: 

790 background.append(bg) 

791 if self.config.doDeblend: 

792 self.deblend.run(exposure=exposure, sources=sourceCat) 

793 self.measurement.run( 

794 measCat=sourceCat, 

795 exposure=exposure, 

796 exposureId=exposureIdInfo.expId 

797 ) 

798 self.postCalibrationMeasurement.run( 

799 measCat=sourceCat, 

800 exposure=exposure, 

801 exposureId=exposureIdInfo.expId 

802 ) 

803 if self.config.doApCorr: 

804 self.applyApCorr.run( 

805 catalog=sourceCat, 

806 apCorrMap=exposure.getInfo().getApCorrMap() 

807 ) 

808 self.catalogCalculation.run(sourceCat) 

809 

810 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

811 self.copyIcSourceFields(icSourceCat=icSourceCat, 

812 sourceCat=sourceCat) 

813 

814 if self.config.doComputeSummaryStats: 

815 summary = self.computeSummaryStats.run(exposure=exposure, 

816 sources=sourceCat, 

817 background=background) 

818 exposure.getInfo().setSummaryStats(summary) 

819 

820 frame = getDebugFrame(self._display, "calibrate") 

821 if frame: 

822 displayAstrometry( 

823 sourceCat=sourceCat, 

824 exposure=exposure, 

825 matches=astromMatches, 

826 frame=frame, 

827 pause=False, 

828 ) 

829 

830 return pipeBase.Struct( 

831 exposure=exposure, 

832 background=background, 

833 sourceCat=sourceCat, 

834 astromMatches=astromMatches, 

835 matchMeta=matchMeta, 

836 # These are duplicate entries with different names for use with 

837 # gen3 middleware 

838 outputExposure=exposure, 

839 outputCat=sourceCat, 

840 outputBackground=background, 

841 ) 

842 

843 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

844 astromMatches, matchMeta): 

845 """Write output data to the output repository 

846 

847 @param[in] dataRef butler data reference corresponding to a science 

848 image 

849 @param[in] exposure exposure to write 

850 @param[in] background background model for exposure 

851 @param[in] sourceCat catalog of measured sources 

852 @param[in] astromMatches list of source/refObj matches from the 

853 astrometry solver 

854 """ 

855 dataRef.put(sourceCat, "src") 

856 if self.config.doWriteMatches and astromMatches is not None: 

857 normalizedMatches = afwTable.packMatches(astromMatches) 

858 normalizedMatches.table.setMetadata(matchMeta) 

859 dataRef.put(normalizedMatches, "srcMatch") 

860 if self.config.doWriteMatchesDenormalized: 

861 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

862 dataRef.put(denormMatches, "srcMatchFull") 

863 if self.config.doWriteExposure: 

864 dataRef.put(exposure, "calexp") 

865 dataRef.put(background, "calexpBackground") 

866 

867 def getSchemaCatalogs(self): 

868 """Return a dict of empty catalogs for each catalog dataset produced 

869 by this task. 

870 """ 

871 sourceCat = afwTable.SourceCatalog(self.schema) 

872 sourceCat.getTable().setMetadata(self.algMetadata) 

873 return {"src": sourceCat} 

874 

875 def setMetadata(self, exposure, photoRes=None): 

876 """!Set task and exposure metadata 

877 

878 Logs a warning and continues if needed data is missing. 

879 

880 @param[in,out] exposure exposure whose metadata is to be set 

881 @param[in] photoRes results of running photoCal; if None then it was 

882 not run 

883 """ 

884 if photoRes is None: 

885 return 

886 

887 metadata = exposure.getMetadata() 

888 

889 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

890 try: 

891 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

892 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

893 except Exception: 

894 self.log.warning("Could not set normalized MAGZERO in header: no " 

895 "exposure time") 

896 magZero = math.nan 

897 

898 try: 

899 metadata.set('MAGZERO', magZero) 

900 metadata.set('MAGZERO_RMS', photoRes.sigma) 

901 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

902 metadata.set('COLORTERM1', 0.0) 

903 metadata.set('COLORTERM2', 0.0) 

904 metadata.set('COLORTERM3', 0.0) 

905 except Exception as e: 

906 self.log.warning("Could not set exposure metadata: %s", e) 

907 

908 def copyIcSourceFields(self, icSourceCat, sourceCat): 

909 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

910 

911 @param[in] icSourceCat catalog from which to copy fields 

912 @param[in,out] sourceCat catalog to which to copy fields 

913 

914 The fields copied are those specified by `config.icSourceFieldsToCopy` 

915 that actually exist in the schema. This was set up by the constructor 

916 using self.schemaMapper. 

917 """ 

918 if self.schemaMapper is None: 

919 raise RuntimeError("To copy icSource fields you must specify " 

920 "icSourceSchema nd icSourceKeys when " 

921 "constructing this task") 

922 if icSourceCat is None or sourceCat is None: 

923 raise RuntimeError("icSourceCat and sourceCat must both be " 

924 "specified") 

925 if len(self.config.icSourceFieldsToCopy) == 0: 

926 self.log.warning("copyIcSourceFields doing nothing because " 

927 "icSourceFieldsToCopy is empty") 

928 return 

929 

930 mc = afwTable.MatchControl() 

931 mc.findOnlyClosest = False # return all matched objects 

932 matches = afwTable.matchXy(icSourceCat, sourceCat, 

933 self.config.matchRadiusPix, mc) 

934 if self.config.doDeblend: 

935 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

936 # if deblended, keep children 

937 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

938 

939 # Because we had to allow multiple matches to handle parents, we now 

940 # need to prune to the best matches 

941 # closest matches as a dict of icSourceCat source ID: 

942 # (icSourceCat source, sourceCat source, distance in pixels) 

943 bestMatches = {} 

944 for m0, m1, d in matches: 

945 id0 = m0.getId() 

946 match = bestMatches.get(id0) 

947 if match is None or d <= match[2]: 

948 bestMatches[id0] = (m0, m1, d) 

949 matches = list(bestMatches.values()) 

950 

951 # Check that no sourceCat sources are listed twice (we already know 

952 # that each match has a unique icSourceCat source ID, due to using 

953 # that ID as the key in bestMatches) 

954 numMatches = len(matches) 

955 numUniqueSources = len(set(m[1].getId() for m in matches)) 

956 if numUniqueSources != numMatches: 

957 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

958 "sources", numMatches, numUniqueSources) 

959 

960 self.log.info("Copying flags from icSourceCat to sourceCat for " 

961 "%d sources", numMatches) 

962 

963 # For each match: set the calibSourceKey flag and copy the desired 

964 # fields 

965 for icSrc, src, d in matches: 

966 src.setFlag(self.calibSourceKey, True) 

967 # src.assign copies the footprint from icSrc, which we don't want 

968 # (DM-407) 

969 # so set icSrc's footprint to src's footprint before src.assign, 

970 # then restore it 

971 icSrcFootprint = icSrc.getFootprint() 

972 try: 

973 icSrc.setFootprint(src.getFootprint()) 

974 src.assign(icSrc, self.schemaMapper) 

975 finally: 

976 icSrc.setFootprint(icSrcFootprint)