Coverage for python/lsst/pipe/tasks/calibrate.py: 20%

287 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-17 10:15 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import warnings 

26import numpy as np 

27 

28from lsstDebug import getDebugFrame 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32import lsst.afw.table as afwTable 

33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

34from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

35import lsst.daf.base as dafBase 

36from lsst.afw.math import BackgroundList 

37from lsst.afw.table import SourceTable 

38from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

39from lsst.meas.base import (SingleFrameMeasurementTask, 

40 ApplyApCorrTask, 

41 CatalogCalculationTask, 

42 IdGenerator, 

43 DetectorVisitIdGeneratorConfig) 

44from lsst.meas.deblender import SourceDeblendTask 

45from lsst.utils.timer import timeMethod 

46from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

47from .fakes import BaseFakeSourcesTask 

48from .photoCal import PhotoCalTask 

49from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

50 

51 

52class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

53 defaultTemplates={}): 

54 

55 icSourceSchema = cT.InitInput( 

56 doc="Schema produced by characterize image task, used to initialize this task", 

57 name="icSrc_schema", 

58 storageClass="SourceCatalog", 

59 ) 

60 

61 outputSchema = cT.InitOutput( 

62 doc="Schema after CalibrateTask has been initialized", 

63 name="src_schema", 

64 storageClass="SourceCatalog", 

65 ) 

66 

67 exposure = cT.Input( 

68 doc="Input image to calibrate", 

69 name="icExp", 

70 storageClass="ExposureF", 

71 dimensions=("instrument", "visit", "detector"), 

72 ) 

73 

74 background = cT.Input( 

75 doc="Backgrounds determined by characterize task", 

76 name="icExpBackground", 

77 storageClass="Background", 

78 dimensions=("instrument", "visit", "detector"), 

79 ) 

80 

81 icSourceCat = cT.Input( 

82 doc="Source catalog created by characterize task", 

83 name="icSrc", 

84 storageClass="SourceCatalog", 

85 dimensions=("instrument", "visit", "detector"), 

86 ) 

87 

88 astromRefCat = cT.PrerequisiteInput( 

89 doc="Reference catalog to use for astrometry", 

90 name="gaia_dr2_20200414", 

91 storageClass="SimpleCatalog", 

92 dimensions=("skypix",), 

93 deferLoad=True, 

94 multiple=True, 

95 ) 

96 

97 photoRefCat = cT.PrerequisiteInput( 

98 doc="Reference catalog to use for photometric calibration", 

99 name="ps1_pv3_3pi_20170110", 

100 storageClass="SimpleCatalog", 

101 dimensions=("skypix",), 

102 deferLoad=True, 

103 multiple=True 

104 ) 

105 

106 outputExposure = cT.Output( 

107 doc="Exposure after running calibration task", 

108 name="calexp", 

109 storageClass="ExposureF", 

110 dimensions=("instrument", "visit", "detector"), 

111 ) 

112 

113 outputCat = cT.Output( 

114 doc="Source catalog produced in calibrate task", 

115 name="src", 

116 storageClass="SourceCatalog", 

117 dimensions=("instrument", "visit", "detector"), 

118 ) 

119 

120 outputBackground = cT.Output( 

121 doc="Background models estimated in calibration task", 

122 name="calexpBackground", 

123 storageClass="Background", 

124 dimensions=("instrument", "visit", "detector"), 

125 ) 

126 

127 matches = cT.Output( 

128 doc="Source/refObj matches from the astrometry solver", 

129 name="srcMatch", 

130 storageClass="Catalog", 

131 dimensions=("instrument", "visit", "detector"), 

132 ) 

133 

134 matchesDenormalized = cT.Output( 

135 doc="Denormalized matches from astrometry solver", 

136 name="srcMatchFull", 

137 storageClass="Catalog", 

138 dimensions=("instrument", "visit", "detector"), 

139 ) 

140 

141 def __init__(self, *, config=None): 

142 super().__init__(config=config) 

143 

144 if config.doAstrometry is False: 

145 self.prerequisiteInputs.remove("astromRefCat") 

146 if config.doPhotoCal is False: 

147 self.prerequisiteInputs.remove("photoRefCat") 

148 

149 if config.doWriteMatches is False or config.doAstrometry is False: 

150 self.outputs.remove("matches") 

151 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

152 self.outputs.remove("matchesDenormalized") 

153 

154 

155class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

156 """Config for CalibrateTask.""" 

157 

158 doWrite = pexConfig.Field( 

159 dtype=bool, 

160 default=True, 

161 doc="Save calibration results?", 

162 ) 

163 doWriteHeavyFootprintsInSources = pexConfig.Field( 

164 dtype=bool, 

165 default=True, 

166 doc="Include HeavyFootprint data in source table? If false then heavy " 

167 "footprints are saved as normal footprints, which saves some space" 

168 ) 

169 doWriteMatches = pexConfig.Field( 

170 dtype=bool, 

171 default=True, 

172 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

173 ) 

174 doWriteMatchesDenormalized = pexConfig.Field( 

175 dtype=bool, 

176 default=False, 

177 doc=("Write reference matches in denormalized format? " 

178 "This format uses more disk space, but is more convenient to " 

179 "read. Ignored if doWriteMatches=False or doWrite=False."), 

180 ) 

181 doAstrometry = pexConfig.Field( 

182 dtype=bool, 

183 default=True, 

184 doc="Perform astrometric calibration?", 

185 ) 

186 astromRefObjLoader = pexConfig.ConfigField( 

187 dtype=LoadReferenceObjectsConfig, 

188 doc="reference object loader for astrometric calibration", 

189 ) 

190 photoRefObjLoader = pexConfig.ConfigField( 

191 dtype=LoadReferenceObjectsConfig, 

192 doc="reference object loader for photometric calibration", 

193 ) 

194 astrometry = pexConfig.ConfigurableField( 

195 target=AstrometryTask, 

196 doc="Perform astrometric calibration to refine the WCS", 

197 ) 

198 requireAstrometry = pexConfig.Field( 

199 dtype=bool, 

200 default=True, 

201 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

202 "false."), 

203 ) 

204 doPhotoCal = pexConfig.Field( 

205 dtype=bool, 

206 default=True, 

207 doc="Perform phometric calibration?", 

208 ) 

209 requirePhotoCal = pexConfig.Field( 

210 dtype=bool, 

211 default=True, 

212 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

213 "false."), 

214 ) 

215 photoCal = pexConfig.ConfigurableField( 

216 target=PhotoCalTask, 

217 doc="Perform photometric calibration", 

218 ) 

219 icSourceFieldsToCopy = pexConfig.ListField( 

220 dtype=str, 

221 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

222 doc=("Fields to copy from the icSource catalog to the output catalog " 

223 "for matching sources Any missing fields will trigger a " 

224 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

225 ) 

226 matchRadiusPix = pexConfig.Field( 

227 dtype=float, 

228 default=3, 

229 doc=("Match radius for matching icSourceCat objects to sourceCat " 

230 "objects (pixels)"), 

231 ) 

232 checkUnitsParseStrict = pexConfig.Field( 

233 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

234 "'warn' or 'silent'"), 

235 dtype=str, 

236 default="raise", 

237 ) 

238 detection = pexConfig.ConfigurableField( 

239 target=SourceDetectionTask, 

240 doc="Detect sources" 

241 ) 

242 doDeblend = pexConfig.Field( 

243 dtype=bool, 

244 default=True, 

245 doc="Run deblender input exposure" 

246 ) 

247 deblend = pexConfig.ConfigurableField( 

248 target=SourceDeblendTask, 

249 doc="Split blended sources into their components" 

250 ) 

251 doSkySources = pexConfig.Field( 

252 dtype=bool, 

253 default=True, 

254 doc="Generate sky sources?", 

255 ) 

256 skySources = pexConfig.ConfigurableField( 

257 target=SkyObjectsTask, 

258 doc="Generate sky sources", 

259 ) 

260 measurement = pexConfig.ConfigurableField( 

261 target=SingleFrameMeasurementTask, 

262 doc="Measure sources" 

263 ) 

264 postCalibrationMeasurement = pexConfig.ConfigurableField( 

265 target=SingleFrameMeasurementTask, 

266 doc="Second round of measurement for plugins that need to be run after photocal" 

267 ) 

268 setPrimaryFlags = pexConfig.ConfigurableField( 

269 target=SetPrimaryFlagsTask, 

270 doc=("Set flags for primary source classification in single frame " 

271 "processing. True if sources are not sky sources and not a parent.") 

272 ) 

273 doApCorr = pexConfig.Field( 

274 dtype=bool, 

275 default=True, 

276 doc="Run subtask to apply aperture correction" 

277 ) 

278 applyApCorr = pexConfig.ConfigurableField( 

279 target=ApplyApCorrTask, 

280 doc="Subtask to apply aperture corrections" 

281 ) 

282 # If doApCorr is False, and the exposure does not have apcorrections 

283 # already applied, the active plugins in catalogCalculation almost 

284 # certainly should not contain the characterization plugin 

285 catalogCalculation = pexConfig.ConfigurableField( 

286 target=CatalogCalculationTask, 

287 doc="Subtask to run catalogCalculation plugins on catalog" 

288 ) 

289 doInsertFakes = pexConfig.Field( 

290 dtype=bool, 

291 default=False, 

292 doc="Run fake sources injection task", 

293 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. " 

294 "Please use ProcessCcdWithFakesTask instead.") 

295 ) 

296 insertFakes = pexConfig.ConfigurableField( 

297 target=BaseFakeSourcesTask, 

298 doc="Injection of fake sources for testing purposes (must be " 

299 "retargeted)", 

300 deprecated=("insertFakes is no longer supported. This config will be removed after v24. " 

301 "Please use ProcessCcdWithFakesTask instead.") 

302 ) 

303 doComputeSummaryStats = pexConfig.Field( 

304 dtype=bool, 

305 default=True, 

306 doc="Run subtask to measure exposure summary statistics?" 

307 ) 

308 computeSummaryStats = pexConfig.ConfigurableField( 

309 target=ComputeExposureSummaryStatsTask, 

310 doc="Subtask to run computeSummaryStats on exposure" 

311 ) 

312 doWriteExposure = pexConfig.Field( 

313 dtype=bool, 

314 default=True, 

315 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

316 "normal calexp but as a fakes_calexp." 

317 ) 

318 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

319 

320 def setDefaults(self): 

321 super().setDefaults() 

322 self.detection.doTempLocalBackground = False 

323 self.deblend.maxFootprintSize = 2000 

324 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

325 self.postCalibrationMeasurement.doReplaceWithNoise = False 

326 for key in self.postCalibrationMeasurement.slots: 

327 setattr(self.postCalibrationMeasurement.slots, key, None) 

328 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

329 # The photoRefCat connection is the name to use for the colorterms. 

330 self.photoCal.photoCatName = self.connections.photoRefCat 

331 

332 

333class CalibrateTask(pipeBase.PipelineTask): 

334 """Calibrate an exposure: measure sources and perform astrometric and 

335 photometric calibration. 

336 

337 Given an exposure with a good PSF model and aperture correction map(e.g. as 

338 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

339 perform the following operations: 

340 - Run detection and measurement 

341 - Run astrometry subtask to fit an improved WCS 

342 - Run photoCal subtask to fit the exposure's photometric zero-point 

343 

344 Parameters 

345 ---------- 

346 butler : `None` 

347 Compatibility parameter. Should always be `None`. 

348 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

349 Unused in gen3: must be `None`. 

350 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

351 Unused in gen3: must be `None`. 

352 icSourceSchema : `lsst.afw.table.Schema`, optional 

353 Schema for the icSource catalog. 

354 initInputs : `dict`, optional 

355 Dictionary that can contain a key ``icSourceSchema`` containing the 

356 input schema. If present will override the value of ``icSourceSchema``. 

357 

358 Raises 

359 ------ 

360 RuntimeError 

361 Raised if any of the following occur: 

362 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

363 - PipelineTask form of this task is initialized with reference object 

364 loaders. 

365 

366 Notes 

367 ----- 

368 Quantities set in exposure Metadata: 

369 

370 MAGZERO_RMS 

371 MAGZERO's RMS == sigma reported by photoCal task 

372 MAGZERO_NOBJ 

373 Number of stars used == ngood reported by photoCal task 

374 COLORTERM1 

375 ?? (always 0.0) 

376 COLORTERM2 

377 ?? (always 0.0) 

378 COLORTERM3 

379 ?? (always 0.0) 

380 

381 Debugging: 

382 CalibrateTask has a debug dictionary containing one key: 

383 

384 calibrate 

385 frame (an int; <= 0 to not display) in which to display the exposure, 

386 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

387 the meaning of the various symbols. 

388 """ 

389 

390 ConfigClass = CalibrateConfig 

391 _DefaultName = "calibrate" 

392 

393 def __init__(self, butler=None, astromRefObjLoader=None, 

394 photoRefObjLoader=None, icSourceSchema=None, 

395 initInputs=None, **kwargs): 

396 super().__init__(**kwargs) 

397 

398 if butler is not None: 

399 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

400 category=FutureWarning, stacklevel=2) 

401 butler = None 

402 

403 if initInputs is not None: 

404 icSourceSchema = initInputs['icSourceSchema'].schema 

405 

406 if icSourceSchema is not None: 

407 # use a schema mapper to avoid copying each field separately 

408 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

409 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

410 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

411 

412 # Add fields to copy from an icSource catalog 

413 # and a field to indicate that the source matched a source in that 

414 # catalog. If any fields are missing then raise an exception, but 

415 # first find all missing fields in order to make the error message 

416 # more useful. 

417 self.calibSourceKey = self.schemaMapper.addOutputField( 

418 afwTable.Field["Flag"]("calib_detected", 

419 "Source was detected as an icSource")) 

420 missingFieldNames = [] 

421 for fieldName in self.config.icSourceFieldsToCopy: 

422 try: 

423 schemaItem = icSourceSchema.find(fieldName) 

424 except Exception: 

425 missingFieldNames.append(fieldName) 

426 else: 

427 # field found; if addMapping fails then raise an exception 

428 self.schemaMapper.addMapping(schemaItem.getKey()) 

429 

430 if missingFieldNames: 

431 raise RuntimeError("isSourceCat is missing fields {} " 

432 "specified in icSourceFieldsToCopy" 

433 .format(missingFieldNames)) 

434 

435 # produce a temporary schema to pass to the subtasks; finalize it 

436 # later 

437 self.schema = self.schemaMapper.editOutputSchema() 

438 else: 

439 self.schemaMapper = None 

440 self.schema = afwTable.SourceTable.makeMinimalSchema() 

441 self.makeSubtask('detection', schema=self.schema) 

442 

443 self.algMetadata = dafBase.PropertyList() 

444 

445 if self.config.doDeblend: 

446 self.makeSubtask("deblend", schema=self.schema) 

447 if self.config.doSkySources: 

448 self.makeSubtask("skySources") 

449 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

450 self.makeSubtask('measurement', schema=self.schema, 

451 algMetadata=self.algMetadata) 

452 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

453 algMetadata=self.algMetadata) 

454 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

455 if self.config.doApCorr: 

456 self.makeSubtask('applyApCorr', schema=self.schema) 

457 self.makeSubtask('catalogCalculation', schema=self.schema) 

458 

459 if self.config.doAstrometry: 

460 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

461 schema=self.schema) 

462 if self.config.doPhotoCal: 

463 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

464 schema=self.schema) 

465 if self.config.doComputeSummaryStats: 

466 self.makeSubtask('computeSummaryStats') 

467 

468 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

469 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

470 "reference object loaders.") 

471 

472 if self.schemaMapper is not None: 

473 # finalize the schema 

474 self.schema = self.schemaMapper.getOutputSchema() 

475 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

476 

477 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

478 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

479 self.outputSchema = sourceCatSchema 

480 

481 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

482 inputs = butlerQC.get(inputRefs) 

483 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

484 

485 if self.config.doAstrometry: 

486 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

487 for ref in inputRefs.astromRefCat], 

488 refCats=inputs.pop('astromRefCat'), 

489 name=self.config.connections.astromRefCat, 

490 config=self.config.astromRefObjLoader, log=self.log) 

491 self.astrometry.setRefObjLoader(refObjLoader) 

492 

493 if self.config.doPhotoCal: 

494 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

495 for ref in inputRefs.photoRefCat], 

496 refCats=inputs.pop('photoRefCat'), 

497 name=self.config.connections.photoRefCat, 

498 config=self.config.photoRefObjLoader, 

499 log=self.log) 

500 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

501 

502 outputs = self.run(**inputs) 

503 

504 if self.config.doWriteMatches and self.config.doAstrometry: 

505 if outputs.astromMatches is not None: 

506 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

507 normalizedMatches.table.setMetadata(outputs.matchMeta) 

508 if self.config.doWriteMatchesDenormalized: 

509 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

510 outputs.matchesDenormalized = denormMatches 

511 outputs.matches = normalizedMatches 

512 else: 

513 del outputRefs.matches 

514 if self.config.doWriteMatchesDenormalized: 

515 del outputRefs.matchesDenormalized 

516 butlerQC.put(outputs, outputRefs) 

517 

518 @timeMethod 

519 def run(self, exposure, exposureIdInfo=None, background=None, 

520 icSourceCat=None, idGenerator=None): 

521 """Calibrate an exposure. 

522 

523 Parameters 

524 ---------- 

525 exposure : `lsst.afw.image.ExposureF` 

526 Exposure to calibrate. 

527 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

528 Exposure ID info. Deprecated in favor of ``idGenerator``, and 

529 ignored if that is provided. 

530 background : `lsst.afw.math.BackgroundList`, optional 

531 Initial model of background already subtracted from exposure. 

532 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

533 SourceCatalog from CharacterizeImageTask from which we can copy 

534 some fields. 

535 idGenerator : `lsst.meas.base.IdGenerator`, optional 

536 Object that generates source IDs and provides RNG seeds. 

537 

538 Returns 

539 ------- 

540 result : `lsst.pipe.base.Struct` 

541 Results as a struct with attributes: 

542 

543 ``exposure`` 

544 Characterized exposure (`lsst.afw.image.ExposureF`). 

545 ``sourceCat`` 

546 Detected sources (`lsst.afw.table.SourceCatalog`). 

547 ``outputBackground`` 

548 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

549 ``astromMatches`` 

550 List of source/ref matches from astrometry solver. 

551 ``matchMeta`` 

552 Metadata from astrometry matches. 

553 ``outputExposure`` 

554 Another reference to ``exposure`` for compatibility. 

555 ``outputCat`` 

556 Another reference to ``sourceCat`` for compatibility. 

557 """ 

558 # detect, deblend and measure sources 

559 if idGenerator is None: 

560 if exposureIdInfo is not None: 

561 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

562 else: 

563 idGenerator = IdGenerator() 

564 

565 if background is None: 

566 background = BackgroundList() 

567 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

568 table.setMetadata(self.algMetadata) 

569 

570 detRes = self.detection.run(table=table, exposure=exposure, 

571 doSmooth=True) 

572 sourceCat = detRes.sources 

573 if detRes.background: 

574 for bg in detRes.background: 

575 background.append(bg) 

576 if self.config.doSkySources: 

577 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

578 if skySourceFootprints: 

579 for foot in skySourceFootprints: 

580 s = sourceCat.addNew() 

581 s.setFootprint(foot) 

582 s.set(self.skySourceKey, True) 

583 if self.config.doDeblend: 

584 self.deblend.run(exposure=exposure, sources=sourceCat) 

585 self.measurement.run( 

586 measCat=sourceCat, 

587 exposure=exposure, 

588 exposureId=idGenerator.catalog_id, 

589 ) 

590 if self.config.doApCorr: 

591 apCorrMap = exposure.getInfo().getApCorrMap() 

592 if apCorrMap is None: 

593 self.log.warning("Image does not have valid aperture correction map for %r; " 

594 "skipping aperture correction", idGenerator) 

595 else: 

596 self.applyApCorr.run( 

597 catalog=sourceCat, 

598 apCorrMap=apCorrMap, 

599 ) 

600 self.catalogCalculation.run(sourceCat) 

601 

602 self.setPrimaryFlags.run(sourceCat) 

603 

604 if icSourceCat is not None and \ 

605 len(self.config.icSourceFieldsToCopy) > 0: 

606 self.copyIcSourceFields(icSourceCat=icSourceCat, 

607 sourceCat=sourceCat) 

608 

609 # TODO DM-11568: this contiguous check-and-copy could go away if we 

610 # reserve enough space during SourceDetection and/or SourceDeblend. 

611 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

612 # contiguity now, so views are preserved from here on. 

613 if not sourceCat.isContiguous(): 

614 sourceCat = sourceCat.copy(deep=True) 

615 

616 # perform astrometry calibration: 

617 # fit an improved WCS and update the exposure's WCS in place 

618 astromMatches = None 

619 matchMeta = None 

620 if self.config.doAstrometry: 

621 astromRes = self.astrometry.run( 

622 exposure=exposure, 

623 sourceCat=sourceCat, 

624 ) 

625 astromMatches = astromRes.matches 

626 matchMeta = astromRes.matchMeta 

627 if exposure.getWcs() is None: 

628 if self.config.requireAstrometry: 

629 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

630 "is True.") 

631 else: 

632 self.log.warning("Unable to perform astrometric calibration for %r but " 

633 "requireAstrometry is False: attempting to proceed...", 

634 idGenerator) 

635 

636 # compute photometric calibration 

637 if self.config.doPhotoCal: 

638 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

639 if self.config.requirePhotoCal: 

640 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

641 "photoCal, but requirePhotoCal is True.") 

642 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

643 "is False, so skipping photometric calibration and setting photoCalib " 

644 "to None. Attempting to proceed...", idGenerator) 

645 exposure.setPhotoCalib(None) 

646 self.setMetadata(exposure=exposure, photoRes=None) 

647 else: 

648 try: 

649 photoRes = self.photoCal.run( 

650 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

651 ) 

652 exposure.setPhotoCalib(photoRes.photoCalib) 

653 # TODO: reword this to phrase it in terms of the 

654 # calibration factor? 

655 self.log.info("Photometric zero-point: %f", 

656 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

657 self.setMetadata(exposure=exposure, photoRes=photoRes) 

658 except Exception as e: 

659 if self.config.requirePhotoCal: 

660 raise 

661 self.log.warning("Unable to perform photometric calibration " 

662 "(%s): attempting to proceed", e) 

663 self.setMetadata(exposure=exposure, photoRes=None) 

664 

665 self.postCalibrationMeasurement.run( 

666 measCat=sourceCat, 

667 exposure=exposure, 

668 exposureId=idGenerator.catalog_id, 

669 ) 

670 

671 if self.config.doComputeSummaryStats: 

672 summary = self.computeSummaryStats.run(exposure=exposure, 

673 sources=sourceCat, 

674 background=background) 

675 exposure.getInfo().setSummaryStats(summary) 

676 

677 frame = getDebugFrame(self._display, "calibrate") 

678 if frame: 

679 displayAstrometry( 

680 sourceCat=sourceCat, 

681 exposure=exposure, 

682 matches=astromMatches, 

683 frame=frame, 

684 pause=False, 

685 ) 

686 

687 return pipeBase.Struct( 

688 sourceCat=sourceCat, 

689 astromMatches=astromMatches, 

690 matchMeta=matchMeta, 

691 outputExposure=exposure, 

692 outputCat=sourceCat, 

693 outputBackground=background, 

694 ) 

695 

696 def setMetadata(self, exposure, photoRes=None): 

697 """Set task and exposure metadata. 

698 

699 Logs a warning continues if needed data is missing. 

700 

701 Parameters 

702 ---------- 

703 exposure : `lsst.afw.image.ExposureF` 

704 Exposure to set metadata on. 

705 photoRes : `lsst.pipe.base.Struct`, optional 

706 Result of running photoCal task. 

707 """ 

708 if photoRes is None: 

709 return 

710 

711 metadata = exposure.getMetadata() 

712 

713 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

714 try: 

715 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

716 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

717 except Exception: 

718 self.log.warning("Could not set normalized MAGZERO in header: no " 

719 "exposure time") 

720 magZero = math.nan 

721 

722 try: 

723 metadata.set('MAGZERO', magZero) 

724 metadata.set('MAGZERO_RMS', photoRes.sigma) 

725 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

726 metadata.set('COLORTERM1', 0.0) 

727 metadata.set('COLORTERM2', 0.0) 

728 metadata.set('COLORTERM3', 0.0) 

729 except Exception as e: 

730 self.log.warning("Could not set exposure metadata: %s", e) 

731 

732 def copyIcSourceFields(self, icSourceCat, sourceCat): 

733 """Match sources in an icSourceCat and a sourceCat and copy fields. 

734 

735 The fields copied are those specified by 

736 ``config.icSourceFieldsToCopy``. 

737 

738 Parameters 

739 ---------- 

740 icSourceCat : `lsst.afw.table.SourceCatalog` 

741 Catalog from which to copy fields. 

742 sourceCat : `lsst.afw.table.SourceCatalog` 

743 Catalog to which to copy fields. 

744 

745 Raises 

746 ------ 

747 RuntimeError 

748 Raised if any of the following occur: 

749 - icSourceSchema and icSourceKeys are not specified. 

750 - icSourceCat and sourceCat are not specified. 

751 - icSourceFieldsToCopy is empty. 

752 """ 

753 if self.schemaMapper is None: 

754 raise RuntimeError("To copy icSource fields you must specify " 

755 "icSourceSchema and icSourceKeys when " 

756 "constructing this task") 

757 if icSourceCat is None or sourceCat is None: 

758 raise RuntimeError("icSourceCat and sourceCat must both be " 

759 "specified") 

760 if len(self.config.icSourceFieldsToCopy) == 0: 

761 self.log.warning("copyIcSourceFields doing nothing because " 

762 "icSourceFieldsToCopy is empty") 

763 return 

764 

765 mc = afwTable.MatchControl() 

766 mc.findOnlyClosest = False # return all matched objects 

767 matches = afwTable.matchXy(icSourceCat, sourceCat, 

768 self.config.matchRadiusPix, mc) 

769 if self.config.doDeblend: 

770 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

771 # if deblended, keep children 

772 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

773 

774 # Because we had to allow multiple matches to handle parents, we now 

775 # need to prune to the best matches 

776 # closest matches as a dict of icSourceCat source ID: 

777 # (icSourceCat source, sourceCat source, distance in pixels) 

778 bestMatches = {} 

779 for m0, m1, d in matches: 

780 id0 = m0.getId() 

781 match = bestMatches.get(id0) 

782 if match is None or d <= match[2]: 

783 bestMatches[id0] = (m0, m1, d) 

784 matches = list(bestMatches.values()) 

785 

786 # Check that no sourceCat sources are listed twice (we already know 

787 # that each match has a unique icSourceCat source ID, due to using 

788 # that ID as the key in bestMatches) 

789 numMatches = len(matches) 

790 numUniqueSources = len(set(m[1].getId() for m in matches)) 

791 if numUniqueSources != numMatches: 

792 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

793 "sources", numMatches, numUniqueSources) 

794 

795 self.log.info("Copying flags from icSourceCat to sourceCat for " 

796 "%d sources", numMatches) 

797 

798 # For each match: set the calibSourceKey flag and copy the desired 

799 # fields 

800 for icSrc, src, d in matches: 

801 src.setFlag(self.calibSourceKey, True) 

802 # src.assign copies the footprint from icSrc, which we don't want 

803 # (DM-407) 

804 # so set icSrc's footprint to src's footprint before src.assign, 

805 # then restore it 

806 icSrcFootprint = icSrc.getFootprint() 

807 try: 

808 icSrc.setFootprint(src.getFootprint()) 

809 src.assign(icSrc, self.schemaMapper) 

810 finally: 

811 icSrc.setFootprint(icSrcFootprint)