Coverage for python/lsst/pipe/tasks/calibrate.py: 20%

283 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-23 02:25 -0800

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import warnings 

26import numpy as np 

27 

28from lsstDebug import getDebugFrame 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32import lsst.afw.table as afwTable 

33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

34from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

35from lsst.obs.base import ExposureIdInfo 

36import lsst.daf.base as dafBase 

37from lsst.afw.math import BackgroundList 

38from lsst.afw.table import SourceTable 

39from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

40from lsst.meas.base import (SingleFrameMeasurementTask, 

41 ApplyApCorrTask, 

42 CatalogCalculationTask) 

43from lsst.meas.deblender import SourceDeblendTask 

44from lsst.utils.timer import timeMethod 

45from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

46from .fakes import BaseFakeSourcesTask 

47from .photoCal import PhotoCalTask 

48from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

49 

50 

51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

52 defaultTemplates={}): 

53 

54 icSourceSchema = cT.InitInput( 

55 doc="Schema produced by characterize image task, used to initialize this task", 

56 name="icSrc_schema", 

57 storageClass="SourceCatalog", 

58 ) 

59 

60 outputSchema = cT.InitOutput( 

61 doc="Schema after CalibrateTask has been initialized", 

62 name="src_schema", 

63 storageClass="SourceCatalog", 

64 ) 

65 

66 exposure = cT.Input( 

67 doc="Input image to calibrate", 

68 name="icExp", 

69 storageClass="ExposureF", 

70 dimensions=("instrument", "visit", "detector"), 

71 ) 

72 

73 background = cT.Input( 

74 doc="Backgrounds determined by characterize task", 

75 name="icExpBackground", 

76 storageClass="Background", 

77 dimensions=("instrument", "visit", "detector"), 

78 ) 

79 

80 icSourceCat = cT.Input( 

81 doc="Source catalog created by characterize task", 

82 name="icSrc", 

83 storageClass="SourceCatalog", 

84 dimensions=("instrument", "visit", "detector"), 

85 ) 

86 

87 astromRefCat = cT.PrerequisiteInput( 

88 doc="Reference catalog to use for astrometry", 

89 name="gaia_dr2_20200414", 

90 storageClass="SimpleCatalog", 

91 dimensions=("skypix",), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 photoRefCat = cT.PrerequisiteInput( 

97 doc="Reference catalog to use for photometric calibration", 

98 name="ps1_pv3_3pi_20170110", 

99 storageClass="SimpleCatalog", 

100 dimensions=("skypix",), 

101 deferLoad=True, 

102 multiple=True 

103 ) 

104 

105 outputExposure = cT.Output( 

106 doc="Exposure after running calibration task", 

107 name="calexp", 

108 storageClass="ExposureF", 

109 dimensions=("instrument", "visit", "detector"), 

110 ) 

111 

112 outputCat = cT.Output( 

113 doc="Source catalog produced in calibrate task", 

114 name="src", 

115 storageClass="SourceCatalog", 

116 dimensions=("instrument", "visit", "detector"), 

117 ) 

118 

119 outputBackground = cT.Output( 

120 doc="Background models estimated in calibration task", 

121 name="calexpBackground", 

122 storageClass="Background", 

123 dimensions=("instrument", "visit", "detector"), 

124 ) 

125 

126 matches = cT.Output( 

127 doc="Source/refObj matches from the astrometry solver", 

128 name="srcMatch", 

129 storageClass="Catalog", 

130 dimensions=("instrument", "visit", "detector"), 

131 ) 

132 

133 matchesDenormalized = cT.Output( 

134 doc="Denormalized matches from astrometry solver", 

135 name="srcMatchFull", 

136 storageClass="Catalog", 

137 dimensions=("instrument", "visit", "detector"), 

138 ) 

139 

140 def __init__(self, *, config=None): 

141 super().__init__(config=config) 

142 

143 if config.doAstrometry is False: 

144 self.prerequisiteInputs.remove("astromRefCat") 

145 if config.doPhotoCal is False: 

146 self.prerequisiteInputs.remove("photoRefCat") 

147 

148 if config.doWriteMatches is False or config.doAstrometry is False: 

149 self.outputs.remove("matches") 

150 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

151 self.outputs.remove("matchesDenormalized") 

152 

153 

154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

155 """Config for CalibrateTask.""" 

156 

157 doWrite = pexConfig.Field( 

158 dtype=bool, 

159 default=True, 

160 doc="Save calibration results?", 

161 ) 

162 doWriteHeavyFootprintsInSources = pexConfig.Field( 

163 dtype=bool, 

164 default=True, 

165 doc="Include HeavyFootprint data in source table? If false then heavy " 

166 "footprints are saved as normal footprints, which saves some space" 

167 ) 

168 doWriteMatches = pexConfig.Field( 

169 dtype=bool, 

170 default=True, 

171 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

172 ) 

173 doWriteMatchesDenormalized = pexConfig.Field( 

174 dtype=bool, 

175 default=False, 

176 doc=("Write reference matches in denormalized format? " 

177 "This format uses more disk space, but is more convenient to " 

178 "read. Ignored if doWriteMatches=False or doWrite=False."), 

179 ) 

180 doAstrometry = pexConfig.Field( 

181 dtype=bool, 

182 default=True, 

183 doc="Perform astrometric calibration?", 

184 ) 

185 astromRefObjLoader = pexConfig.ConfigField( 

186 dtype=LoadReferenceObjectsConfig, 

187 doc="reference object loader for astrometric calibration", 

188 ) 

189 photoRefObjLoader = pexConfig.ConfigField( 

190 dtype=LoadReferenceObjectsConfig, 

191 doc="reference object loader for photometric calibration", 

192 ) 

193 astrometry = pexConfig.ConfigurableField( 

194 target=AstrometryTask, 

195 doc="Perform astrometric calibration to refine the WCS", 

196 ) 

197 requireAstrometry = pexConfig.Field( 

198 dtype=bool, 

199 default=True, 

200 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

201 "false."), 

202 ) 

203 doPhotoCal = pexConfig.Field( 

204 dtype=bool, 

205 default=True, 

206 doc="Perform phometric calibration?", 

207 ) 

208 requirePhotoCal = pexConfig.Field( 

209 dtype=bool, 

210 default=True, 

211 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

212 "false."), 

213 ) 

214 photoCal = pexConfig.ConfigurableField( 

215 target=PhotoCalTask, 

216 doc="Perform photometric calibration", 

217 ) 

218 icSourceFieldsToCopy = pexConfig.ListField( 

219 dtype=str, 

220 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

221 doc=("Fields to copy from the icSource catalog to the output catalog " 

222 "for matching sources Any missing fields will trigger a " 

223 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

224 ) 

225 matchRadiusPix = pexConfig.Field( 

226 dtype=float, 

227 default=3, 

228 doc=("Match radius for matching icSourceCat objects to sourceCat " 

229 "objects (pixels)"), 

230 ) 

231 checkUnitsParseStrict = pexConfig.Field( 

232 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

233 "'warn' or 'silent'"), 

234 dtype=str, 

235 default="raise", 

236 ) 

237 detection = pexConfig.ConfigurableField( 

238 target=SourceDetectionTask, 

239 doc="Detect sources" 

240 ) 

241 doDeblend = pexConfig.Field( 

242 dtype=bool, 

243 default=True, 

244 doc="Run deblender input exposure" 

245 ) 

246 deblend = pexConfig.ConfigurableField( 

247 target=SourceDeblendTask, 

248 doc="Split blended sources into their components" 

249 ) 

250 doSkySources = pexConfig.Field( 

251 dtype=bool, 

252 default=True, 

253 doc="Generate sky sources?", 

254 ) 

255 skySources = pexConfig.ConfigurableField( 

256 target=SkyObjectsTask, 

257 doc="Generate sky sources", 

258 ) 

259 measurement = pexConfig.ConfigurableField( 

260 target=SingleFrameMeasurementTask, 

261 doc="Measure sources" 

262 ) 

263 postCalibrationMeasurement = pexConfig.ConfigurableField( 

264 target=SingleFrameMeasurementTask, 

265 doc="Second round of measurement for plugins that need to be run after photocal" 

266 ) 

267 setPrimaryFlags = pexConfig.ConfigurableField( 

268 target=SetPrimaryFlagsTask, 

269 doc=("Set flags for primary source classification in single frame " 

270 "processing. True if sources are not sky sources and not a parent.") 

271 ) 

272 doApCorr = pexConfig.Field( 

273 dtype=bool, 

274 default=True, 

275 doc="Run subtask to apply aperture correction" 

276 ) 

277 applyApCorr = pexConfig.ConfigurableField( 

278 target=ApplyApCorrTask, 

279 doc="Subtask to apply aperture corrections" 

280 ) 

281 # If doApCorr is False, and the exposure does not have apcorrections 

282 # already applied, the active plugins in catalogCalculation almost 

283 # certainly should not contain the characterization plugin 

284 catalogCalculation = pexConfig.ConfigurableField( 

285 target=CatalogCalculationTask, 

286 doc="Subtask to run catalogCalculation plugins on catalog" 

287 ) 

288 doInsertFakes = pexConfig.Field( 

289 dtype=bool, 

290 default=False, 

291 doc="Run fake sources injection task", 

292 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. " 

293 "Please use ProcessCcdWithFakesTask instead.") 

294 ) 

295 insertFakes = pexConfig.ConfigurableField( 

296 target=BaseFakeSourcesTask, 

297 doc="Injection of fake sources for testing purposes (must be " 

298 "retargeted)", 

299 deprecated=("insertFakes is no longer supported. This config will be removed after v24. " 

300 "Please use ProcessCcdWithFakesTask instead.") 

301 ) 

302 doComputeSummaryStats = pexConfig.Field( 

303 dtype=bool, 

304 default=True, 

305 doc="Run subtask to measure exposure summary statistics?" 

306 ) 

307 computeSummaryStats = pexConfig.ConfigurableField( 

308 target=ComputeExposureSummaryStatsTask, 

309 doc="Subtask to run computeSummaryStats on exposure" 

310 ) 

311 doWriteExposure = pexConfig.Field( 

312 dtype=bool, 

313 default=True, 

314 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

315 "normal calexp but as a fakes_calexp." 

316 ) 

317 

318 def setDefaults(self): 

319 super().setDefaults() 

320 self.detection.doTempLocalBackground = False 

321 self.deblend.maxFootprintSize = 2000 

322 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

323 self.postCalibrationMeasurement.doReplaceWithNoise = False 

324 for key in self.postCalibrationMeasurement.slots: 

325 setattr(self.postCalibrationMeasurement.slots, key, None) 

326 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

327 # The photoRefCat connection is the name to use for the colorterms. 

328 self.photoCal.photoCatName = self.connections.photoRefCat 

329 

330 

331class CalibrateTask(pipeBase.PipelineTask): 

332 """Calibrate an exposure: measure sources and perform astrometric and 

333 photometric calibration. 

334 

335 Given an exposure with a good PSF model and aperture correction map(e.g. as 

336 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

337 perform the following operations: 

338 - Run detection and measurement 

339 - Run astrometry subtask to fit an improved WCS 

340 - Run photoCal subtask to fit the exposure's photometric zero-point 

341 

342 Parameters 

343 ---------- 

344 butler : `None` 

345 Compatibility parameter. Should always be `None`. 

346 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

347 Unused in gen3: must be `None`. 

348 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

349 Unused in gen3: must be `None`. 

350 icSourceSchema : `lsst.afw.table.Schema`, optional 

351 Schema for the icSource catalog. 

352 initInputs : `dict`, optional 

353 Dictionary that can contain a key ``icSourceSchema`` containing the 

354 input schema. If present will override the value of ``icSourceSchema``. 

355 

356 Raises 

357 ------ 

358 RuntimeError 

359 Raised if any of the following occur: 

360 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

361 - PipelineTask form of this task is initialized with reference object 

362 loaders. 

363 

364 Notes 

365 ----- 

366 Quantities set in exposure Metadata: 

367 

368 MAGZERO_RMS 

369 MAGZERO's RMS == sigma reported by photoCal task 

370 MAGZERO_NOBJ 

371 Number of stars used == ngood reported by photoCal task 

372 COLORTERM1 

373 ?? (always 0.0) 

374 COLORTERM2 

375 ?? (always 0.0) 

376 COLORTERM3 

377 ?? (always 0.0) 

378 

379 Debugging: 

380 CalibrateTask has a debug dictionary containing one key: 

381 

382 calibrate 

383 frame (an int; <= 0 to not display) in which to display the exposure, 

384 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

385 the meaning of the various symbols. 

386 """ 

387 

388 ConfigClass = CalibrateConfig 

389 _DefaultName = "calibrate" 

390 

391 def __init__(self, butler=None, astromRefObjLoader=None, 

392 photoRefObjLoader=None, icSourceSchema=None, 

393 initInputs=None, **kwargs): 

394 super().__init__(**kwargs) 

395 

396 if butler is not None: 

397 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

398 category=FutureWarning, stacklevel=2) 

399 butler = None 

400 

401 if initInputs is not None: 

402 icSourceSchema = initInputs['icSourceSchema'].schema 

403 

404 if icSourceSchema is not None: 

405 # use a schema mapper to avoid copying each field separately 

406 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

407 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

408 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

409 

410 # Add fields to copy from an icSource catalog 

411 # and a field to indicate that the source matched a source in that 

412 # catalog. If any fields are missing then raise an exception, but 

413 # first find all missing fields in order to make the error message 

414 # more useful. 

415 self.calibSourceKey = self.schemaMapper.addOutputField( 

416 afwTable.Field["Flag"]("calib_detected", 

417 "Source was detected as an icSource")) 

418 missingFieldNames = [] 

419 for fieldName in self.config.icSourceFieldsToCopy: 

420 try: 

421 schemaItem = icSourceSchema.find(fieldName) 

422 except Exception: 

423 missingFieldNames.append(fieldName) 

424 else: 

425 # field found; if addMapping fails then raise an exception 

426 self.schemaMapper.addMapping(schemaItem.getKey()) 

427 

428 if missingFieldNames: 

429 raise RuntimeError("isSourceCat is missing fields {} " 

430 "specified in icSourceFieldsToCopy" 

431 .format(missingFieldNames)) 

432 

433 # produce a temporary schema to pass to the subtasks; finalize it 

434 # later 

435 self.schema = self.schemaMapper.editOutputSchema() 

436 else: 

437 self.schemaMapper = None 

438 self.schema = afwTable.SourceTable.makeMinimalSchema() 

439 self.makeSubtask('detection', schema=self.schema) 

440 

441 self.algMetadata = dafBase.PropertyList() 

442 

443 if self.config.doDeblend: 

444 self.makeSubtask("deblend", schema=self.schema) 

445 if self.config.doSkySources: 

446 self.makeSubtask("skySources") 

447 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

448 self.makeSubtask('measurement', schema=self.schema, 

449 algMetadata=self.algMetadata) 

450 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

451 algMetadata=self.algMetadata) 

452 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

453 if self.config.doApCorr: 

454 self.makeSubtask('applyApCorr', schema=self.schema) 

455 self.makeSubtask('catalogCalculation', schema=self.schema) 

456 

457 if self.config.doAstrometry: 

458 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

459 schema=self.schema) 

460 if self.config.doPhotoCal: 

461 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

462 schema=self.schema) 

463 if self.config.doComputeSummaryStats: 

464 self.makeSubtask('computeSummaryStats') 

465 

466 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

467 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

468 "reference object loaders.") 

469 

470 if self.schemaMapper is not None: 

471 # finalize the schema 

472 self.schema = self.schemaMapper.getOutputSchema() 

473 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

474 

475 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

476 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

477 self.outputSchema = sourceCatSchema 

478 

479 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

480 inputs = butlerQC.get(inputRefs) 

481 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

482 

483 if self.config.doAstrometry: 

484 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

485 for ref in inputRefs.astromRefCat], 

486 refCats=inputs.pop('astromRefCat'), 

487 name=self.config.connections.astromRefCat, 

488 config=self.config.astromRefObjLoader, log=self.log) 

489 self.astrometry.setRefObjLoader(refObjLoader) 

490 

491 if self.config.doPhotoCal: 

492 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

493 for ref in inputRefs.photoRefCat], 

494 refCats=inputs.pop('photoRefCat'), 

495 name=self.config.connections.photoRefCat, 

496 config=self.config.photoRefObjLoader, 

497 log=self.log) 

498 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

499 

500 outputs = self.run(**inputs) 

501 

502 if self.config.doWriteMatches and self.config.doAstrometry: 

503 if outputs.astromMatches is not None: 

504 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

505 normalizedMatches.table.setMetadata(outputs.matchMeta) 

506 if self.config.doWriteMatchesDenormalized: 

507 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

508 outputs.matchesDenormalized = denormMatches 

509 outputs.matches = normalizedMatches 

510 else: 

511 del outputRefs.matches 

512 if self.config.doWriteMatchesDenormalized: 

513 del outputRefs.matchesDenormalized 

514 butlerQC.put(outputs, outputRefs) 

515 

516 @timeMethod 

517 def run(self, exposure, exposureIdInfo=None, background=None, 

518 icSourceCat=None): 

519 """Calibrate an exposure. 

520 

521 Parameters 

522 ---------- 

523 exposure : `lsst.afw.image.ExposureF` 

524 Exposure to calibrate. 

525 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

526 Exposure ID info. If not provided, returned SourceCatalog IDs will 

527 not be globally unique. 

528 background : `lsst.afw.math.BackgroundList`, optional 

529 Initial model of background already subtracted from exposure. 

530 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

531 SourceCatalog from CharacterizeImageTask from which we can copy 

532 some fields. 

533 

534 Returns 

535 ------- 

536 result : `lsst.pipe.base.Struct` 

537 Results as a struct with attributes: 

538 

539 ``exposure`` 

540 Characterized exposure (`lsst.afw.image.ExposureF`). 

541 ``sourceCat`` 

542 Detected sources (`lsst.afw.table.SourceCatalog`). 

543 ``outputBackground`` 

544 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

545 ``astromMatches`` 

546 List of source/ref matches from astrometry solver. 

547 ``matchMeta`` 

548 Metadata from astrometry matches. 

549 ``outputExposure`` 

550 Another reference to ``exposure`` for compatibility. 

551 ``outputCat`` 

552 Another reference to ``sourceCat`` for compatibility. 

553 """ 

554 # detect, deblend and measure sources 

555 if exposureIdInfo is None: 

556 exposureIdInfo = ExposureIdInfo() 

557 

558 if background is None: 

559 background = BackgroundList() 

560 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

561 table = SourceTable.make(self.schema, sourceIdFactory) 

562 table.setMetadata(self.algMetadata) 

563 

564 detRes = self.detection.run(table=table, exposure=exposure, 

565 doSmooth=True) 

566 sourceCat = detRes.sources 

567 if detRes.fpSets.background: 

568 for bg in detRes.fpSets.background: 

569 background.append(bg) 

570 if self.config.doSkySources: 

571 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

572 if skySourceFootprints: 

573 for foot in skySourceFootprints: 

574 s = sourceCat.addNew() 

575 s.setFootprint(foot) 

576 s.set(self.skySourceKey, True) 

577 if self.config.doDeblend: 

578 self.deblend.run(exposure=exposure, sources=sourceCat) 

579 self.measurement.run( 

580 measCat=sourceCat, 

581 exposure=exposure, 

582 exposureId=exposureIdInfo.expId 

583 ) 

584 if self.config.doApCorr: 

585 self.applyApCorr.run( 

586 catalog=sourceCat, 

587 apCorrMap=exposure.getInfo().getApCorrMap() 

588 ) 

589 self.catalogCalculation.run(sourceCat) 

590 

591 self.setPrimaryFlags.run(sourceCat) 

592 

593 if icSourceCat is not None and \ 

594 len(self.config.icSourceFieldsToCopy) > 0: 

595 self.copyIcSourceFields(icSourceCat=icSourceCat, 

596 sourceCat=sourceCat) 

597 

598 # TODO DM-11568: this contiguous check-and-copy could go away if we 

599 # reserve enough space during SourceDetection and/or SourceDeblend. 

600 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

601 # contiguity now, so views are preserved from here on. 

602 if not sourceCat.isContiguous(): 

603 sourceCat = sourceCat.copy(deep=True) 

604 

605 # perform astrometry calibration: 

606 # fit an improved WCS and update the exposure's WCS in place 

607 astromMatches = None 

608 matchMeta = None 

609 if self.config.doAstrometry: 

610 astromRes = self.astrometry.run( 

611 exposure=exposure, 

612 sourceCat=sourceCat, 

613 ) 

614 astromMatches = astromRes.matches 

615 matchMeta = astromRes.matchMeta 

616 if exposure.getWcs() is None: 

617 if self.config.requireAstrometry: 

618 raise RuntimeError(f"WCS fit failed for {exposureIdInfo.expId} and requireAstrometry " 

619 "is True.") 

620 else: 

621 self.log.warning("Unable to perform astrometric calibration for %r but " 

622 "requireAstrometry is False: attempting to proceed...", 

623 exposureIdInfo.expId) 

624 

625 # compute photometric calibration 

626 if self.config.doPhotoCal: 

627 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

628 if self.config.requirePhotoCal: 

629 raise RuntimeError(f"Astrometry failed for {exposureIdInfo.expId}, so cannot do " 

630 "photoCal, but requirePhotoCal is True.") 

631 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

632 "is False, so skipping photometric calibration and setting photoCalib " 

633 "to None. Attempting to proceed...", exposureIdInfo.expId) 

634 exposure.setPhotoCalib(None) 

635 self.setMetadata(exposure=exposure, photoRes=None) 

636 else: 

637 try: 

638 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

639 exposure.setPhotoCalib(photoRes.photoCalib) 

640 # TODO: reword this to phrase it in terms of the 

641 # calibration factor? 

642 self.log.info("Photometric zero-point: %f", 

643 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

644 self.setMetadata(exposure=exposure, photoRes=photoRes) 

645 except Exception as e: 

646 if self.config.requirePhotoCal: 

647 raise 

648 self.log.warning("Unable to perform photometric calibration " 

649 "(%s): attempting to proceed", e) 

650 self.setMetadata(exposure=exposure, photoRes=None) 

651 

652 self.postCalibrationMeasurement.run( 

653 measCat=sourceCat, 

654 exposure=exposure, 

655 exposureId=exposureIdInfo.expId 

656 ) 

657 

658 if self.config.doComputeSummaryStats: 

659 summary = self.computeSummaryStats.run(exposure=exposure, 

660 sources=sourceCat, 

661 background=background) 

662 exposure.getInfo().setSummaryStats(summary) 

663 

664 frame = getDebugFrame(self._display, "calibrate") 

665 if frame: 

666 displayAstrometry( 

667 sourceCat=sourceCat, 

668 exposure=exposure, 

669 matches=astromMatches, 

670 frame=frame, 

671 pause=False, 

672 ) 

673 

674 return pipeBase.Struct( 

675 sourceCat=sourceCat, 

676 astromMatches=astromMatches, 

677 matchMeta=matchMeta, 

678 outputExposure=exposure, 

679 outputCat=sourceCat, 

680 outputBackground=background, 

681 ) 

682 

683 def setMetadata(self, exposure, photoRes=None): 

684 """Set task and exposure metadata. 

685 

686 Logs a warning continues if needed data is missing. 

687 

688 Parameters 

689 ---------- 

690 exposure : `lsst.afw.image.ExposureF` 

691 Exposure to set metadata on. 

692 photoRes : `lsst.pipe.base.Struct`, optional 

693 Result of running photoCal task. 

694 """ 

695 if photoRes is None: 

696 return 

697 

698 metadata = exposure.getMetadata() 

699 

700 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

701 try: 

702 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

703 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

704 except Exception: 

705 self.log.warning("Could not set normalized MAGZERO in header: no " 

706 "exposure time") 

707 magZero = math.nan 

708 

709 try: 

710 metadata.set('MAGZERO', magZero) 

711 metadata.set('MAGZERO_RMS', photoRes.sigma) 

712 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

713 metadata.set('COLORTERM1', 0.0) 

714 metadata.set('COLORTERM2', 0.0) 

715 metadata.set('COLORTERM3', 0.0) 

716 except Exception as e: 

717 self.log.warning("Could not set exposure metadata: %s", e) 

718 

719 def copyIcSourceFields(self, icSourceCat, sourceCat): 

720 """Match sources in an icSourceCat and a sourceCat and copy fields. 

721 

722 The fields copied are those specified by 

723 ``config.icSourceFieldsToCopy``. 

724 

725 Parameters 

726 ---------- 

727 icSourceCat : `lsst.afw.table.SourceCatalog` 

728 Catalog from which to copy fields. 

729 sourceCat : `lsst.afw.table.SourceCatalog` 

730 Catalog to which to copy fields. 

731 

732 Raises 

733 ------ 

734 RuntimeError 

735 Raised if any of the following occur: 

736 - icSourceSchema and icSourceKeys are not specified. 

737 - icSourceCat and sourceCat are not specified. 

738 - icSourceFieldsToCopy is empty. 

739 """ 

740 if self.schemaMapper is None: 

741 raise RuntimeError("To copy icSource fields you must specify " 

742 "icSourceSchema and icSourceKeys when " 

743 "constructing this task") 

744 if icSourceCat is None or sourceCat is None: 

745 raise RuntimeError("icSourceCat and sourceCat must both be " 

746 "specified") 

747 if len(self.config.icSourceFieldsToCopy) == 0: 

748 self.log.warning("copyIcSourceFields doing nothing because " 

749 "icSourceFieldsToCopy is empty") 

750 return 

751 

752 mc = afwTable.MatchControl() 

753 mc.findOnlyClosest = False # return all matched objects 

754 matches = afwTable.matchXy(icSourceCat, sourceCat, 

755 self.config.matchRadiusPix, mc) 

756 if self.config.doDeblend: 

757 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

758 # if deblended, keep children 

759 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

760 

761 # Because we had to allow multiple matches to handle parents, we now 

762 # need to prune to the best matches 

763 # closest matches as a dict of icSourceCat source ID: 

764 # (icSourceCat source, sourceCat source, distance in pixels) 

765 bestMatches = {} 

766 for m0, m1, d in matches: 

767 id0 = m0.getId() 

768 match = bestMatches.get(id0) 

769 if match is None or d <= match[2]: 

770 bestMatches[id0] = (m0, m1, d) 

771 matches = list(bestMatches.values()) 

772 

773 # Check that no sourceCat sources are listed twice (we already know 

774 # that each match has a unique icSourceCat source ID, due to using 

775 # that ID as the key in bestMatches) 

776 numMatches = len(matches) 

777 numUniqueSources = len(set(m[1].getId() for m in matches)) 

778 if numUniqueSources != numMatches: 

779 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

780 "sources", numMatches, numUniqueSources) 

781 

782 self.log.info("Copying flags from icSourceCat to sourceCat for " 

783 "%d sources", numMatches) 

784 

785 # For each match: set the calibSourceKey flag and copy the desired 

786 # fields 

787 for icSrc, src, d in matches: 

788 src.setFlag(self.calibSourceKey, True) 

789 # src.assign copies the footprint from icSrc, which we don't want 

790 # (DM-407) 

791 # so set icSrc's footprint to src's footprint before src.assign, 

792 # then restore it 

793 icSrcFootprint = icSrc.getFootprint() 

794 try: 

795 icSrc.setFootprint(src.getFootprint()) 

796 src.assign(icSrc, self.schemaMapper) 

797 finally: 

798 icSrc.setFootprint(icSrcFootprint)