Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

279 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-27 12:31 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import numpy as np 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

33from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

34import lsst.daf.base as dafBase 

35from lsst.afw.math import BackgroundList 

36from lsst.afw.table import SourceTable 

37from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

38from lsst.meas.base import (SingleFrameMeasurementTask, 

39 ApplyApCorrTask, 

40 CatalogCalculationTask, 

41 IdGenerator, 

42 DetectorVisitIdGeneratorConfig) 

43from lsst.meas.deblender import SourceDeblendTask 

44from lsst.utils.timer import timeMethod 

45from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

46from .photoCal import PhotoCalTask 

47from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="gaia_dr3_20230707", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="ps1_pv3_3pi_20170110", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask.""" 

155 

156 doWrite = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Save calibration results?", 

160 ) 

161 doWriteHeavyFootprintsInSources = pexConfig.Field( 

162 dtype=bool, 

163 default=True, 

164 doc="Include HeavyFootprint data in source table? If false then heavy " 

165 "footprints are saved as normal footprints, which saves some space" 

166 ) 

167 doWriteMatches = pexConfig.Field( 

168 dtype=bool, 

169 default=True, 

170 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

171 ) 

172 doWriteMatchesDenormalized = pexConfig.Field( 

173 dtype=bool, 

174 default=True, 

175 doc=("Write reference matches in denormalized format? " 

176 "This format uses more disk space, but is more convenient to " 

177 "read for debugging. Ignored if doWriteMatches=False or doWrite=False."), 

178 ) 

179 doAstrometry = pexConfig.Field( 

180 dtype=bool, 

181 default=True, 

182 doc="Perform astrometric calibration?", 

183 ) 

184 astromRefObjLoader = pexConfig.ConfigField( 

185 dtype=LoadReferenceObjectsConfig, 

186 doc="reference object loader for astrometric calibration", 

187 ) 

188 photoRefObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 doc="reference object loader for photometric calibration", 

191 ) 

192 astrometry = pexConfig.ConfigurableField( 

193 target=AstrometryTask, 

194 doc="Perform astrometric calibration to refine the WCS", 

195 ) 

196 requireAstrometry = pexConfig.Field( 

197 dtype=bool, 

198 default=True, 

199 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

200 "false."), 

201 ) 

202 doPhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc="Perform phometric calibration?", 

206 ) 

207 requirePhotoCal = pexConfig.Field( 

208 dtype=bool, 

209 default=True, 

210 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

211 "false."), 

212 ) 

213 photoCal = pexConfig.ConfigurableField( 

214 target=PhotoCalTask, 

215 doc="Perform photometric calibration", 

216 ) 

217 icSourceFieldsToCopy = pexConfig.ListField( 

218 dtype=str, 

219 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

220 doc=("Fields to copy from the icSource catalog to the output catalog " 

221 "for matching sources Any missing fields will trigger a " 

222 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

223 ) 

224 matchRadiusPix = pexConfig.Field( 

225 dtype=float, 

226 default=3, 

227 doc=("Match radius for matching icSourceCat objects to sourceCat " 

228 "objects (pixels)"), 

229 ) 

230 checkUnitsParseStrict = pexConfig.Field( 

231 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

232 "'warn' or 'silent'"), 

233 dtype=str, 

234 default="raise", 

235 ) 

236 detection = pexConfig.ConfigurableField( 

237 target=SourceDetectionTask, 

238 doc="Detect sources" 

239 ) 

240 doDeblend = pexConfig.Field( 

241 dtype=bool, 

242 default=True, 

243 doc="Run deblender input exposure" 

244 ) 

245 deblend = pexConfig.ConfigurableField( 

246 target=SourceDeblendTask, 

247 doc="Split blended sources into their components" 

248 ) 

249 doSkySources = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Generate sky sources?", 

253 ) 

254 skySources = pexConfig.ConfigurableField( 

255 target=SkyObjectsTask, 

256 doc="Generate sky sources", 

257 ) 

258 measurement = pexConfig.ConfigurableField( 

259 target=SingleFrameMeasurementTask, 

260 doc="Measure sources" 

261 ) 

262 postCalibrationMeasurement = pexConfig.ConfigurableField( 

263 target=SingleFrameMeasurementTask, 

264 doc="Second round of measurement for plugins that need to be run after photocal" 

265 ) 

266 setPrimaryFlags = pexConfig.ConfigurableField( 

267 target=SetPrimaryFlagsTask, 

268 doc=("Set flags for primary source classification in single frame " 

269 "processing. True if sources are not sky sources and not a parent.") 

270 ) 

271 doApCorr = pexConfig.Field( 

272 dtype=bool, 

273 default=True, 

274 doc="Run subtask to apply aperture correction" 

275 ) 

276 applyApCorr = pexConfig.ConfigurableField( 

277 target=ApplyApCorrTask, 

278 doc="Subtask to apply aperture corrections" 

279 ) 

280 # If doApCorr is False, and the exposure does not have apcorrections 

281 # already applied, the active plugins in catalogCalculation almost 

282 # certainly should not contain the characterization plugin 

283 catalogCalculation = pexConfig.ConfigurableField( 

284 target=CatalogCalculationTask, 

285 doc="Subtask to run catalogCalculation plugins on catalog" 

286 ) 

287 doComputeSummaryStats = pexConfig.Field( 

288 dtype=bool, 

289 default=True, 

290 doc="Run subtask to measure exposure summary statistics?" 

291 ) 

292 computeSummaryStats = pexConfig.ConfigurableField( 

293 target=ComputeExposureSummaryStatsTask, 

294 doc="Subtask to run computeSummaryStats on exposure" 

295 ) 

296 doWriteExposure = pexConfig.Field( 

297 dtype=bool, 

298 default=True, 

299 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

300 "normal calexp but as a fakes_calexp." 

301 ) 

302 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

303 

304 def setDefaults(self): 

305 super().setDefaults() 

306 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

307 self.postCalibrationMeasurement.doReplaceWithNoise = False 

308 for key in self.postCalibrationMeasurement.slots: 

309 setattr(self.postCalibrationMeasurement.slots, key, None) 

310 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

311 # The photoRefCat connection is the name to use for the colorterms. 

312 self.photoCal.photoCatName = self.connections.photoRefCat 

313 

314 

315class CalibrateTask(pipeBase.PipelineTask): 

316 """Calibrate an exposure: measure sources and perform astrometric and 

317 photometric calibration. 

318 

319 Given an exposure with a good PSF model and aperture correction map(e.g. as 

320 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

321 perform the following operations: 

322 - Run detection and measurement 

323 - Run astrometry subtask to fit an improved WCS 

324 - Run photoCal subtask to fit the exposure's photometric zero-point 

325 

326 Parameters 

327 ---------- 

328 butler : `None` 

329 Compatibility parameter. Should always be `None`. 

330 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

331 Unused in gen3: must be `None`. 

332 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

333 Unused in gen3: must be `None`. 

334 icSourceSchema : `lsst.afw.table.Schema`, optional 

335 Schema for the icSource catalog. 

336 initInputs : `dict`, optional 

337 Dictionary that can contain a key ``icSourceSchema`` containing the 

338 input schema. If present will override the value of ``icSourceSchema``. 

339 

340 Raises 

341 ------ 

342 RuntimeError 

343 Raised if any of the following occur: 

344 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

345 - PipelineTask form of this task is initialized with reference object 

346 loaders. 

347 

348 Notes 

349 ----- 

350 Quantities set in exposure Metadata: 

351 

352 MAGZERO_RMS 

353 MAGZERO's RMS == sigma reported by photoCal task 

354 MAGZERO_NOBJ 

355 Number of stars used == ngood reported by photoCal task 

356 COLORTERM1 

357 ?? (always 0.0) 

358 COLORTERM2 

359 ?? (always 0.0) 

360 COLORTERM3 

361 ?? (always 0.0) 

362 

363 Debugging: 

364 CalibrateTask has a debug dictionary containing one key: 

365 

366 calibrate 

367 frame (an int; <= 0 to not display) in which to display the exposure, 

368 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

369 the meaning of the various symbols. 

370 """ 

371 

372 ConfigClass = CalibrateConfig 

373 _DefaultName = "calibrate" 

374 

375 def __init__(self, astromRefObjLoader=None, 

376 photoRefObjLoader=None, icSourceSchema=None, 

377 initInputs=None, **kwargs): 

378 super().__init__(**kwargs) 

379 

380 if initInputs is not None: 

381 icSourceSchema = initInputs['icSourceSchema'].schema 

382 

383 if icSourceSchema is not None: 

384 # use a schema mapper to avoid copying each field separately 

385 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

386 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

387 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

388 

389 # Add fields to copy from an icSource catalog 

390 # and a field to indicate that the source matched a source in that 

391 # catalog. If any fields are missing then raise an exception, but 

392 # first find all missing fields in order to make the error message 

393 # more useful. 

394 self.calibSourceKey = self.schemaMapper.addOutputField( 

395 afwTable.Field["Flag"]("calib_detected", 

396 "Source was detected as an icSource")) 

397 missingFieldNames = [] 

398 for fieldName in self.config.icSourceFieldsToCopy: 

399 try: 

400 schemaItem = icSourceSchema.find(fieldName) 

401 except Exception: 

402 missingFieldNames.append(fieldName) 

403 else: 

404 # field found; if addMapping fails then raise an exception 

405 self.schemaMapper.addMapping(schemaItem.getKey()) 

406 

407 if missingFieldNames: 

408 raise RuntimeError("isSourceCat is missing fields {} " 

409 "specified in icSourceFieldsToCopy" 

410 .format(missingFieldNames)) 

411 

412 # produce a temporary schema to pass to the subtasks; finalize it 

413 # later 

414 self.schema = self.schemaMapper.editOutputSchema() 

415 else: 

416 self.schemaMapper = None 

417 self.schema = afwTable.SourceTable.makeMinimalSchema() 

418 afwTable.CoordKey.addErrorFields(self.schema) 

419 self.makeSubtask('detection', schema=self.schema) 

420 

421 self.algMetadata = dafBase.PropertyList() 

422 

423 if self.config.doDeblend: 

424 self.makeSubtask("deblend", schema=self.schema) 

425 if self.config.doSkySources: 

426 self.makeSubtask("skySources") 

427 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

428 self.makeSubtask('measurement', schema=self.schema, 

429 algMetadata=self.algMetadata) 

430 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

431 algMetadata=self.algMetadata) 

432 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

433 if self.config.doApCorr: 

434 self.makeSubtask('applyApCorr', schema=self.schema) 

435 self.makeSubtask('catalogCalculation', schema=self.schema) 

436 

437 if self.config.doAstrometry: 

438 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

439 schema=self.schema) 

440 if self.config.doPhotoCal: 

441 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

442 schema=self.schema) 

443 if self.config.doComputeSummaryStats: 

444 self.makeSubtask('computeSummaryStats') 

445 

446 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

447 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

448 "reference object loaders.") 

449 

450 if self.schemaMapper is not None: 

451 # finalize the schema 

452 self.schema = self.schemaMapper.getOutputSchema() 

453 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

454 

455 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

456 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

457 self.outputSchema = sourceCatSchema 

458 

459 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

460 inputs = butlerQC.get(inputRefs) 

461 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

462 

463 if self.config.doAstrometry: 

464 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

465 for ref in inputRefs.astromRefCat], 

466 refCats=inputs.pop('astromRefCat'), 

467 name=self.config.connections.astromRefCat, 

468 config=self.config.astromRefObjLoader, log=self.log) 

469 self.astrometry.setRefObjLoader(refObjLoader) 

470 

471 if self.config.doPhotoCal: 

472 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

473 for ref in inputRefs.photoRefCat], 

474 refCats=inputs.pop('photoRefCat'), 

475 name=self.config.connections.photoRefCat, 

476 config=self.config.photoRefObjLoader, 

477 log=self.log) 

478 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

479 

480 outputs = self.run(**inputs) 

481 

482 if self.config.doWriteMatches and self.config.doAstrometry: 

483 if outputs.astromMatches is not None: 

484 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

485 normalizedMatches.table.setMetadata(outputs.matchMeta) 

486 if self.config.doWriteMatchesDenormalized: 

487 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

488 outputs.matchesDenormalized = denormMatches 

489 outputs.matches = normalizedMatches 

490 else: 

491 del outputRefs.matches 

492 if self.config.doWriteMatchesDenormalized: 

493 del outputRefs.matchesDenormalized 

494 butlerQC.put(outputs, outputRefs) 

495 

496 @timeMethod 

497 def run(self, exposure, exposureIdInfo=None, background=None, 

498 icSourceCat=None, idGenerator=None): 

499 """Calibrate an exposure. 

500 

501 Parameters 

502 ---------- 

503 exposure : `lsst.afw.image.ExposureF` 

504 Exposure to calibrate. 

505 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

506 Exposure ID info. Deprecated in favor of ``idGenerator``, and 

507 ignored if that is provided. 

508 background : `lsst.afw.math.BackgroundList`, optional 

509 Initial model of background already subtracted from exposure. 

510 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

511 SourceCatalog from CharacterizeImageTask from which we can copy 

512 some fields. 

513 idGenerator : `lsst.meas.base.IdGenerator`, optional 

514 Object that generates source IDs and provides RNG seeds. 

515 

516 Returns 

517 ------- 

518 result : `lsst.pipe.base.Struct` 

519 Results as a struct with attributes: 

520 

521 ``exposure`` 

522 Characterized exposure (`lsst.afw.image.ExposureF`). 

523 ``sourceCat`` 

524 Detected sources (`lsst.afw.table.SourceCatalog`). 

525 ``outputBackground`` 

526 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

527 ``astromMatches`` 

528 List of source/ref matches from astrometry solver. 

529 ``matchMeta`` 

530 Metadata from astrometry matches. 

531 ``outputExposure`` 

532 Another reference to ``exposure`` for compatibility. 

533 ``outputCat`` 

534 Another reference to ``sourceCat`` for compatibility. 

535 """ 

536 # detect, deblend and measure sources 

537 if idGenerator is None: 

538 if exposureIdInfo is not None: 

539 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

540 else: 

541 idGenerator = IdGenerator() 

542 

543 if background is None: 

544 background = BackgroundList() 

545 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

546 table.setMetadata(self.algMetadata) 

547 

548 detRes = self.detection.run(table=table, exposure=exposure, 

549 doSmooth=True) 

550 sourceCat = detRes.sources 

551 if detRes.background: 

552 for bg in detRes.background: 

553 background.append(bg) 

554 if self.config.doSkySources: 

555 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

556 if skySourceFootprints: 

557 for foot in skySourceFootprints: 

558 s = sourceCat.addNew() 

559 s.setFootprint(foot) 

560 s.set(self.skySourceKey, True) 

561 if self.config.doDeblend: 

562 self.deblend.run(exposure=exposure, sources=sourceCat) 

563 self.measurement.run( 

564 measCat=sourceCat, 

565 exposure=exposure, 

566 exposureId=idGenerator.catalog_id, 

567 ) 

568 if self.config.doApCorr: 

569 apCorrMap = exposure.getInfo().getApCorrMap() 

570 if apCorrMap is None: 

571 self.log.warning("Image does not have valid aperture correction map for %r; " 

572 "skipping aperture correction", idGenerator) 

573 else: 

574 self.applyApCorr.run( 

575 catalog=sourceCat, 

576 apCorrMap=apCorrMap, 

577 ) 

578 self.catalogCalculation.run(sourceCat) 

579 

580 self.setPrimaryFlags.run(sourceCat) 

581 

582 if icSourceCat is not None and \ 

583 len(self.config.icSourceFieldsToCopy) > 0: 

584 self.copyIcSourceFields(icSourceCat=icSourceCat, 

585 sourceCat=sourceCat) 

586 

587 # TODO DM-11568: this contiguous check-and-copy could go away if we 

588 # reserve enough space during SourceDetection and/or SourceDeblend. 

589 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

590 # contiguity now, so views are preserved from here on. 

591 if not sourceCat.isContiguous(): 

592 sourceCat = sourceCat.copy(deep=True) 

593 

594 # perform astrometry calibration: 

595 # fit an improved WCS and update the exposure's WCS in place 

596 astromMatches = None 

597 matchMeta = None 

598 if self.config.doAstrometry: 

599 astromRes = self.astrometry.run( 

600 exposure=exposure, 

601 sourceCat=sourceCat, 

602 ) 

603 astromMatches = astromRes.matches 

604 matchMeta = astromRes.matchMeta 

605 if exposure.getWcs() is None: 

606 if self.config.requireAstrometry: 

607 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

608 "is True.") 

609 else: 

610 self.log.warning("Unable to perform astrometric calibration for %r but " 

611 "requireAstrometry is False: attempting to proceed...", 

612 idGenerator) 

613 

614 # compute photometric calibration 

615 if self.config.doPhotoCal: 

616 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

617 if self.config.requirePhotoCal: 

618 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

619 "photoCal, but requirePhotoCal is True.") 

620 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

621 "is False, so skipping photometric calibration and setting photoCalib " 

622 "to None. Attempting to proceed...", idGenerator) 

623 exposure.setPhotoCalib(None) 

624 self.setMetadata(exposure=exposure, photoRes=None) 

625 else: 

626 try: 

627 photoRes = self.photoCal.run( 

628 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

629 ) 

630 exposure.setPhotoCalib(photoRes.photoCalib) 

631 # TODO: reword this to phrase it in terms of the 

632 # calibration factor? 

633 self.log.info("Photometric zero-point: %f", 

634 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

635 self.setMetadata(exposure=exposure, photoRes=photoRes) 

636 except Exception as e: 

637 if self.config.requirePhotoCal: 

638 raise 

639 self.log.warning("Unable to perform photometric calibration " 

640 "(%s): attempting to proceed", e) 

641 self.setMetadata(exposure=exposure, photoRes=None) 

642 

643 self.postCalibrationMeasurement.run( 

644 measCat=sourceCat, 

645 exposure=exposure, 

646 exposureId=idGenerator.catalog_id, 

647 ) 

648 

649 if self.config.doComputeSummaryStats: 

650 summary = self.computeSummaryStats.run(exposure=exposure, 

651 sources=sourceCat, 

652 background=background) 

653 exposure.getInfo().setSummaryStats(summary) 

654 

655 frame = getDebugFrame(self._display, "calibrate") 

656 if frame: 

657 displayAstrometry( 

658 sourceCat=sourceCat, 

659 exposure=exposure, 

660 matches=astromMatches, 

661 frame=frame, 

662 pause=False, 

663 ) 

664 

665 return pipeBase.Struct( 

666 sourceCat=sourceCat, 

667 astromMatches=astromMatches, 

668 matchMeta=matchMeta, 

669 outputExposure=exposure, 

670 outputCat=sourceCat, 

671 outputBackground=background, 

672 ) 

673 

674 def setMetadata(self, exposure, photoRes=None): 

675 """Set task and exposure metadata. 

676 

677 Logs a warning continues if needed data is missing. 

678 

679 Parameters 

680 ---------- 

681 exposure : `lsst.afw.image.ExposureF` 

682 Exposure to set metadata on. 

683 photoRes : `lsst.pipe.base.Struct`, optional 

684 Result of running photoCal task. 

685 """ 

686 if photoRes is None: 

687 return 

688 

689 metadata = exposure.getMetadata() 

690 

691 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

692 try: 

693 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

694 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

695 except Exception: 

696 self.log.warning("Could not set normalized MAGZERO in header: no " 

697 "exposure time") 

698 magZero = math.nan 

699 

700 try: 

701 metadata.set('MAGZERO', magZero) 

702 metadata.set('MAGZERO_RMS', photoRes.sigma) 

703 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

704 metadata.set('COLORTERM1', 0.0) 

705 metadata.set('COLORTERM2', 0.0) 

706 metadata.set('COLORTERM3', 0.0) 

707 except Exception as e: 

708 self.log.warning("Could not set exposure metadata: %s", e) 

709 

710 def copyIcSourceFields(self, icSourceCat, sourceCat): 

711 """Match sources in an icSourceCat and a sourceCat and copy fields. 

712 

713 The fields copied are those specified by 

714 ``config.icSourceFieldsToCopy``. 

715 

716 Parameters 

717 ---------- 

718 icSourceCat : `lsst.afw.table.SourceCatalog` 

719 Catalog from which to copy fields. 

720 sourceCat : `lsst.afw.table.SourceCatalog` 

721 Catalog to which to copy fields. 

722 

723 Raises 

724 ------ 

725 RuntimeError 

726 Raised if any of the following occur: 

727 - icSourceSchema and icSourceKeys are not specified. 

728 - icSourceCat and sourceCat are not specified. 

729 - icSourceFieldsToCopy is empty. 

730 """ 

731 if self.schemaMapper is None: 

732 raise RuntimeError("To copy icSource fields you must specify " 

733 "icSourceSchema and icSourceKeys when " 

734 "constructing this task") 

735 if icSourceCat is None or sourceCat is None: 

736 raise RuntimeError("icSourceCat and sourceCat must both be " 

737 "specified") 

738 if len(self.config.icSourceFieldsToCopy) == 0: 

739 self.log.warning("copyIcSourceFields doing nothing because " 

740 "icSourceFieldsToCopy is empty") 

741 return 

742 

743 mc = afwTable.MatchControl() 

744 mc.findOnlyClosest = False # return all matched objects 

745 matches = afwTable.matchXy(icSourceCat, sourceCat, 

746 self.config.matchRadiusPix, mc) 

747 if self.config.doDeblend: 

748 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

749 # if deblended, keep children 

750 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

751 

752 # Because we had to allow multiple matches to handle parents, we now 

753 # need to prune to the best matches 

754 # closest matches as a dict of icSourceCat source ID: 

755 # (icSourceCat source, sourceCat source, distance in pixels) 

756 bestMatches = {} 

757 for m0, m1, d in matches: 

758 id0 = m0.getId() 

759 match = bestMatches.get(id0) 

760 if match is None or d <= match[2]: 

761 bestMatches[id0] = (m0, m1, d) 

762 matches = list(bestMatches.values()) 

763 

764 # Check that no sourceCat sources are listed twice (we already know 

765 # that each match has a unique icSourceCat source ID, due to using 

766 # that ID as the key in bestMatches) 

767 numMatches = len(matches) 

768 numUniqueSources = len(set(m[1].getId() for m in matches)) 

769 if numUniqueSources != numMatches: 

770 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

771 "sources", numMatches, numUniqueSources) 

772 

773 self.log.info("Copying flags from icSourceCat to sourceCat for " 

774 "%d sources", numMatches) 

775 

776 # For each match: set the calibSourceKey flag and copy the desired 

777 # fields 

778 for icSrc, src, d in matches: 

779 src.setFlag(self.calibSourceKey, True) 

780 # src.assign copies the footprint from icSrc, which we don't want 

781 # (DM-407) 

782 # so set icSrc's footprint to src's footprint before src.assign, 

783 # then restore it 

784 icSrcFootprint = icSrc.getFootprint() 

785 try: 

786 icSrc.setFootprint(src.getFootprint()) 

787 src.assign(icSrc, self.schemaMapper) 

788 finally: 

789 icSrc.setFootprint(icSrcFootprint)