Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

278 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-16 09:33 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import numpy as np 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

33from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

34import lsst.daf.base as dafBase 

35from lsst.afw.math import BackgroundList 

36from lsst.afw.table import SourceTable 

37from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader, SetPrimaryFlagsTask 

38from lsst.meas.base import (SingleFrameMeasurementTask, 

39 ApplyApCorrTask, 

40 CatalogCalculationTask, 

41 IdGenerator, 

42 DetectorVisitIdGeneratorConfig) 

43from lsst.meas.deblender import SourceDeblendTask 

44from lsst.utils.timer import timeMethod 

45from .photoCal import PhotoCalTask 

46from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

47 

48 

49class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

50 defaultTemplates={}): 

51 

52 icSourceSchema = cT.InitInput( 

53 doc="Schema produced by characterize image task, used to initialize this task", 

54 name="icSrc_schema", 

55 storageClass="SourceCatalog", 

56 ) 

57 

58 outputSchema = cT.InitOutput( 

59 doc="Schema after CalibrateTask has been initialized", 

60 name="src_schema", 

61 storageClass="SourceCatalog", 

62 ) 

63 

64 exposure = cT.Input( 

65 doc="Input image to calibrate", 

66 name="icExp", 

67 storageClass="ExposureF", 

68 dimensions=("instrument", "visit", "detector"), 

69 ) 

70 

71 background = cT.Input( 

72 doc="Backgrounds determined by characterize task", 

73 name="icExpBackground", 

74 storageClass="Background", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 

78 icSourceCat = cT.Input( 

79 doc="Source catalog created by characterize task", 

80 name="icSrc", 

81 storageClass="SourceCatalog", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 

85 astromRefCat = cT.PrerequisiteInput( 

86 doc="Reference catalog to use for astrometry", 

87 name="gaia_dr3_20230707", 

88 storageClass="SimpleCatalog", 

89 dimensions=("skypix",), 

90 deferLoad=True, 

91 multiple=True, 

92 ) 

93 

94 photoRefCat = cT.PrerequisiteInput( 

95 doc="Reference catalog to use for photometric calibration", 

96 name="ps1_pv3_3pi_20170110", 

97 storageClass="SimpleCatalog", 

98 dimensions=("skypix",), 

99 deferLoad=True, 

100 multiple=True 

101 ) 

102 

103 outputExposure = cT.Output( 

104 doc="Exposure after running calibration task", 

105 name="calexp", 

106 storageClass="ExposureF", 

107 dimensions=("instrument", "visit", "detector"), 

108 ) 

109 

110 outputCat = cT.Output( 

111 doc="Source catalog produced in calibrate task", 

112 name="src", 

113 storageClass="SourceCatalog", 

114 dimensions=("instrument", "visit", "detector"), 

115 ) 

116 

117 outputBackground = cT.Output( 

118 doc="Background models estimated in calibration task", 

119 name="calexpBackground", 

120 storageClass="Background", 

121 dimensions=("instrument", "visit", "detector"), 

122 ) 

123 

124 matches = cT.Output( 

125 doc="Source/refObj matches from the astrometry solver", 

126 name="srcMatch", 

127 storageClass="Catalog", 

128 dimensions=("instrument", "visit", "detector"), 

129 ) 

130 

131 matchesDenormalized = cT.Output( 

132 doc="Denormalized matches from astrometry solver", 

133 name="srcMatchFull", 

134 storageClass="Catalog", 

135 dimensions=("instrument", "visit", "detector"), 

136 ) 

137 

138 def __init__(self, *, config=None): 

139 super().__init__(config=config) 

140 

141 if config.doAstrometry is False: 

142 self.prerequisiteInputs.remove("astromRefCat") 

143 if config.doPhotoCal is False: 

144 self.prerequisiteInputs.remove("photoRefCat") 

145 

146 if config.doWriteMatches is False or config.doAstrometry is False: 

147 self.outputs.remove("matches") 

148 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

149 self.outputs.remove("matchesDenormalized") 

150 

151 

152class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

153 """Config for CalibrateTask.""" 

154 

155 doWrite = pexConfig.Field( 

156 dtype=bool, 

157 default=True, 

158 doc="Save calibration results?", 

159 ) 

160 doWriteHeavyFootprintsInSources = pexConfig.Field( 

161 dtype=bool, 

162 default=True, 

163 doc="Include HeavyFootprint data in source table? If false then heavy " 

164 "footprints are saved as normal footprints, which saves some space" 

165 ) 

166 doWriteMatches = pexConfig.Field( 

167 dtype=bool, 

168 default=True, 

169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

170 ) 

171 doWriteMatchesDenormalized = pexConfig.Field( 

172 dtype=bool, 

173 default=True, 

174 doc=("Write reference matches in denormalized format? " 

175 "This format uses more disk space, but is more convenient to " 

176 "read for debugging. Ignored if doWriteMatches=False or doWrite=False."), 

177 ) 

178 doAstrometry = pexConfig.Field( 

179 dtype=bool, 

180 default=True, 

181 doc="Perform astrometric calibration?", 

182 ) 

183 astromRefObjLoader = pexConfig.ConfigField( 

184 dtype=LoadReferenceObjectsConfig, 

185 doc="reference object loader for astrometric calibration", 

186 ) 

187 photoRefObjLoader = pexConfig.ConfigField( 

188 dtype=LoadReferenceObjectsConfig, 

189 doc="reference object loader for photometric calibration", 

190 ) 

191 astrometry = pexConfig.ConfigurableField( 

192 target=AstrometryTask, 

193 doc="Perform astrometric calibration to refine the WCS", 

194 ) 

195 requireAstrometry = pexConfig.Field( 

196 dtype=bool, 

197 default=True, 

198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

199 "false."), 

200 ) 

201 doPhotoCal = pexConfig.Field( 

202 dtype=bool, 

203 default=True, 

204 doc="Perform phometric calibration?", 

205 ) 

206 requirePhotoCal = pexConfig.Field( 

207 dtype=bool, 

208 default=True, 

209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

210 "false."), 

211 ) 

212 photoCal = pexConfig.ConfigurableField( 

213 target=PhotoCalTask, 

214 doc="Perform photometric calibration", 

215 ) 

216 icSourceFieldsToCopy = pexConfig.ListField( 

217 dtype=str, 

218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

219 doc=("Fields to copy from the icSource catalog to the output catalog " 

220 "for matching sources Any missing fields will trigger a " 

221 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

222 ) 

223 matchRadiusPix = pexConfig.Field( 

224 dtype=float, 

225 default=3, 

226 doc=("Match radius for matching icSourceCat objects to sourceCat " 

227 "objects (pixels)"), 

228 ) 

229 checkUnitsParseStrict = pexConfig.Field( 

230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

231 "'warn' or 'silent'"), 

232 dtype=str, 

233 default="raise", 

234 ) 

235 detection = pexConfig.ConfigurableField( 

236 target=SourceDetectionTask, 

237 doc="Detect sources" 

238 ) 

239 doDeblend = pexConfig.Field( 

240 dtype=bool, 

241 default=True, 

242 doc="Run deblender input exposure" 

243 ) 

244 deblend = pexConfig.ConfigurableField( 

245 target=SourceDeblendTask, 

246 doc="Split blended sources into their components" 

247 ) 

248 doSkySources = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Generate sky sources?", 

252 ) 

253 skySources = pexConfig.ConfigurableField( 

254 target=SkyObjectsTask, 

255 doc="Generate sky sources", 

256 ) 

257 measurement = pexConfig.ConfigurableField( 

258 target=SingleFrameMeasurementTask, 

259 doc="Measure sources" 

260 ) 

261 postCalibrationMeasurement = pexConfig.ConfigurableField( 

262 target=SingleFrameMeasurementTask, 

263 doc="Second round of measurement for plugins that need to be run after photocal" 

264 ) 

265 setPrimaryFlags = pexConfig.ConfigurableField( 

266 target=SetPrimaryFlagsTask, 

267 doc=("Set flags for primary source classification in single frame " 

268 "processing. True if sources are not sky sources and not a parent.") 

269 ) 

270 doApCorr = pexConfig.Field( 

271 dtype=bool, 

272 default=True, 

273 doc="Run subtask to apply aperture correction" 

274 ) 

275 applyApCorr = pexConfig.ConfigurableField( 

276 target=ApplyApCorrTask, 

277 doc="Subtask to apply aperture corrections" 

278 ) 

279 # If doApCorr is False, and the exposure does not have apcorrections 

280 # already applied, the active plugins in catalogCalculation almost 

281 # certainly should not contain the characterization plugin 

282 catalogCalculation = pexConfig.ConfigurableField( 

283 target=CatalogCalculationTask, 

284 doc="Subtask to run catalogCalculation plugins on catalog" 

285 ) 

286 doComputeSummaryStats = pexConfig.Field( 

287 dtype=bool, 

288 default=True, 

289 doc="Run subtask to measure exposure summary statistics?" 

290 ) 

291 computeSummaryStats = pexConfig.ConfigurableField( 

292 target=ComputeExposureSummaryStatsTask, 

293 doc="Subtask to run computeSummaryStats on exposure" 

294 ) 

295 doWriteExposure = pexConfig.Field( 

296 dtype=bool, 

297 default=True, 

298 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

299 "normal calexp but as a fakes_calexp." 

300 ) 

301 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

302 

303 def setDefaults(self): 

304 super().setDefaults() 

305 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

306 self.postCalibrationMeasurement.doReplaceWithNoise = False 

307 for key in self.postCalibrationMeasurement.slots: 

308 setattr(self.postCalibrationMeasurement.slots, key, None) 

309 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

310 # The photoRefCat connection is the name to use for the colorterms. 

311 self.photoCal.photoCatName = self.connections.photoRefCat 

312 

313 # Keep track of which footprints contain streaks 

314 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

315 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

316 

317 

318class CalibrateTask(pipeBase.PipelineTask): 

319 """Calibrate an exposure: measure sources and perform astrometric and 

320 photometric calibration. 

321 

322 Given an exposure with a good PSF model and aperture correction map(e.g. as 

323 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

324 perform the following operations: 

325 - Run detection and measurement 

326 - Run astrometry subtask to fit an improved WCS 

327 - Run photoCal subtask to fit the exposure's photometric zero-point 

328 

329 Parameters 

330 ---------- 

331 butler : `None` 

332 Compatibility parameter. Should always be `None`. 

333 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

334 Unused in gen3: must be `None`. 

335 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

336 Unused in gen3: must be `None`. 

337 icSourceSchema : `lsst.afw.table.Schema`, optional 

338 Schema for the icSource catalog. 

339 initInputs : `dict`, optional 

340 Dictionary that can contain a key ``icSourceSchema`` containing the 

341 input schema. If present will override the value of ``icSourceSchema``. 

342 

343 Raises 

344 ------ 

345 RuntimeError 

346 Raised if any of the following occur: 

347 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

348 - PipelineTask form of this task is initialized with reference object 

349 loaders. 

350 

351 Notes 

352 ----- 

353 Quantities set in exposure Metadata: 

354 

355 MAGZERO_RMS 

356 MAGZERO's RMS == sigma reported by photoCal task 

357 MAGZERO_NOBJ 

358 Number of stars used == ngood reported by photoCal task 

359 COLORTERM1 

360 ?? (always 0.0) 

361 COLORTERM2 

362 ?? (always 0.0) 

363 COLORTERM3 

364 ?? (always 0.0) 

365 

366 Debugging: 

367 CalibrateTask has a debug dictionary containing one key: 

368 

369 calibrate 

370 frame (an int; <= 0 to not display) in which to display the exposure, 

371 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

372 the meaning of the various symbols. 

373 """ 

374 

375 ConfigClass = CalibrateConfig 

376 _DefaultName = "calibrate" 

377 

378 def __init__(self, astromRefObjLoader=None, 

379 photoRefObjLoader=None, icSourceSchema=None, 

380 initInputs=None, **kwargs): 

381 super().__init__(**kwargs) 

382 

383 if initInputs is not None: 

384 icSourceSchema = initInputs['icSourceSchema'].schema 

385 

386 if icSourceSchema is not None: 

387 # use a schema mapper to avoid copying each field separately 

388 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

389 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

390 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

391 

392 # Add fields to copy from an icSource catalog 

393 # and a field to indicate that the source matched a source in that 

394 # catalog. If any fields are missing then raise an exception, but 

395 # first find all missing fields in order to make the error message 

396 # more useful. 

397 self.calibSourceKey = self.schemaMapper.addOutputField( 

398 afwTable.Field["Flag"]("calib_detected", 

399 "Source was detected as an icSource")) 

400 missingFieldNames = [] 

401 for fieldName in self.config.icSourceFieldsToCopy: 

402 try: 

403 schemaItem = icSourceSchema.find(fieldName) 

404 except Exception: 

405 missingFieldNames.append(fieldName) 

406 else: 

407 # field found; if addMapping fails then raise an exception 

408 self.schemaMapper.addMapping(schemaItem.getKey()) 

409 

410 if missingFieldNames: 

411 raise RuntimeError("isSourceCat is missing fields {} " 

412 "specified in icSourceFieldsToCopy" 

413 .format(missingFieldNames)) 

414 

415 # produce a temporary schema to pass to the subtasks; finalize it 

416 # later 

417 self.schema = self.schemaMapper.editOutputSchema() 

418 else: 

419 self.schemaMapper = None 

420 self.schema = afwTable.SourceTable.makeMinimalSchema() 

421 afwTable.CoordKey.addErrorFields(self.schema) 

422 self.makeSubtask('detection', schema=self.schema) 

423 

424 self.algMetadata = dafBase.PropertyList() 

425 

426 if self.config.doDeblend: 

427 self.makeSubtask("deblend", schema=self.schema) 

428 if self.config.doSkySources: 

429 self.makeSubtask("skySources") 

430 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

431 self.makeSubtask('measurement', schema=self.schema, 

432 algMetadata=self.algMetadata) 

433 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

434 algMetadata=self.algMetadata) 

435 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

436 if self.config.doApCorr: 

437 self.makeSubtask('applyApCorr', schema=self.schema) 

438 self.makeSubtask('catalogCalculation', schema=self.schema) 

439 

440 if self.config.doAstrometry: 

441 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

442 schema=self.schema) 

443 if self.config.doPhotoCal: 

444 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

445 schema=self.schema) 

446 if self.config.doComputeSummaryStats: 

447 self.makeSubtask('computeSummaryStats') 

448 

449 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

450 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

451 "reference object loaders.") 

452 

453 if self.schemaMapper is not None: 

454 # finalize the schema 

455 self.schema = self.schemaMapper.getOutputSchema() 

456 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

457 

458 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

459 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

460 self.outputSchema = sourceCatSchema 

461 

462 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

463 inputs = butlerQC.get(inputRefs) 

464 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

465 

466 if self.config.doAstrometry: 

467 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

468 for ref in inputRefs.astromRefCat], 

469 refCats=inputs.pop('astromRefCat'), 

470 name=self.config.connections.astromRefCat, 

471 config=self.config.astromRefObjLoader, log=self.log) 

472 self.astrometry.setRefObjLoader(refObjLoader) 

473 

474 if self.config.doPhotoCal: 

475 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

476 for ref in inputRefs.photoRefCat], 

477 refCats=inputs.pop('photoRefCat'), 

478 name=self.config.connections.photoRefCat, 

479 config=self.config.photoRefObjLoader, 

480 log=self.log) 

481 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

482 

483 outputs = self.run(**inputs) 

484 

485 if self.config.doWriteMatches and self.config.doAstrometry: 

486 if outputs.astromMatches is not None: 

487 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

488 normalizedMatches.table.setMetadata(outputs.matchMeta) 

489 if self.config.doWriteMatchesDenormalized: 

490 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

491 outputs.matchesDenormalized = denormMatches 

492 outputs.matches = normalizedMatches 

493 else: 

494 del outputRefs.matches 

495 if self.config.doWriteMatchesDenormalized: 

496 del outputRefs.matchesDenormalized 

497 butlerQC.put(outputs, outputRefs) 

498 

499 @timeMethod 

500 def run(self, exposure, background=None, 

501 icSourceCat=None, idGenerator=None): 

502 """Calibrate an exposure. 

503 

504 Parameters 

505 ---------- 

506 exposure : `lsst.afw.image.ExposureF` 

507 Exposure to calibrate. 

508 background : `lsst.afw.math.BackgroundList`, optional 

509 Initial model of background already subtracted from exposure. 

510 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

511 SourceCatalog from CharacterizeImageTask from which we can copy 

512 some fields. 

513 idGenerator : `lsst.meas.base.IdGenerator`, optional 

514 Object that generates source IDs and provides RNG seeds. 

515 

516 Returns 

517 ------- 

518 result : `lsst.pipe.base.Struct` 

519 Results as a struct with attributes: 

520 

521 ``exposure`` 

522 Characterized exposure (`lsst.afw.image.ExposureF`). 

523 ``sourceCat`` 

524 Detected sources (`lsst.afw.table.SourceCatalog`). 

525 ``outputBackground`` 

526 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

527 ``astromMatches`` 

528 List of source/ref matches from astrometry solver. 

529 ``matchMeta`` 

530 Metadata from astrometry matches. 

531 ``outputExposure`` 

532 Another reference to ``exposure`` for compatibility. 

533 ``outputCat`` 

534 Another reference to ``sourceCat`` for compatibility. 

535 """ 

536 # detect, deblend and measure sources 

537 if idGenerator is None: 

538 idGenerator = IdGenerator() 

539 

540 if background is None: 

541 background = BackgroundList() 

542 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

543 table.setMetadata(self.algMetadata) 

544 

545 detRes = self.detection.run(table=table, exposure=exposure, 

546 doSmooth=True) 

547 sourceCat = detRes.sources 

548 if detRes.background: 

549 for bg in detRes.background: 

550 background.append(bg) 

551 if self.config.doSkySources: 

552 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

553 if skySourceFootprints: 

554 for foot in skySourceFootprints: 

555 s = sourceCat.addNew() 

556 s.setFootprint(foot) 

557 s.set(self.skySourceKey, True) 

558 if self.config.doDeblend: 

559 self.deblend.run(exposure=exposure, sources=sourceCat) 

560 self.measurement.run( 

561 measCat=sourceCat, 

562 exposure=exposure, 

563 exposureId=idGenerator.catalog_id, 

564 ) 

565 if self.config.doApCorr: 

566 apCorrMap = exposure.getInfo().getApCorrMap() 

567 if apCorrMap is None: 

568 self.log.warning("Image does not have valid aperture correction map for %r; " 

569 "skipping aperture correction", idGenerator) 

570 else: 

571 self.applyApCorr.run( 

572 catalog=sourceCat, 

573 apCorrMap=apCorrMap, 

574 ) 

575 self.catalogCalculation.run(sourceCat) 

576 

577 self.setPrimaryFlags.run(sourceCat) 

578 

579 if icSourceCat is not None and \ 

580 len(self.config.icSourceFieldsToCopy) > 0: 

581 self.copyIcSourceFields(icSourceCat=icSourceCat, 

582 sourceCat=sourceCat) 

583 

584 # TODO DM-11568: this contiguous check-and-copy could go away if we 

585 # reserve enough space during SourceDetection and/or SourceDeblend. 

586 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

587 # contiguity now, so views are preserved from here on. 

588 if not sourceCat.isContiguous(): 

589 sourceCat = sourceCat.copy(deep=True) 

590 

591 # perform astrometry calibration: 

592 # fit an improved WCS and update the exposure's WCS in place 

593 astromMatches = None 

594 matchMeta = None 

595 if self.config.doAstrometry: 

596 astromRes = self.astrometry.run( 

597 exposure=exposure, 

598 sourceCat=sourceCat, 

599 ) 

600 astromMatches = astromRes.matches 

601 matchMeta = astromRes.matchMeta 

602 if exposure.getWcs() is None: 

603 if self.config.requireAstrometry: 

604 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

605 "is True.") 

606 else: 

607 self.log.warning("Unable to perform astrometric calibration for %r but " 

608 "requireAstrometry is False: attempting to proceed...", 

609 idGenerator) 

610 

611 # compute photometric calibration 

612 if self.config.doPhotoCal: 

613 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

614 if self.config.requirePhotoCal: 

615 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

616 "photoCal, but requirePhotoCal is True.") 

617 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

618 "is False, so skipping photometric calibration and setting photoCalib " 

619 "to None. Attempting to proceed...", idGenerator) 

620 exposure.setPhotoCalib(None) 

621 self.setMetadata(exposure=exposure, photoRes=None) 

622 else: 

623 try: 

624 photoRes = self.photoCal.run( 

625 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

626 ) 

627 exposure.setPhotoCalib(photoRes.photoCalib) 

628 # TODO: reword this to phrase it in terms of the 

629 # calibration factor? 

630 self.log.info("Photometric zero-point: %f", 

631 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

632 self.setMetadata(exposure=exposure, photoRes=photoRes) 

633 except Exception as e: 

634 if self.config.requirePhotoCal: 

635 raise 

636 self.log.warning("Unable to perform photometric calibration " 

637 "(%s): attempting to proceed", e) 

638 self.setMetadata(exposure=exposure, photoRes=None) 

639 

640 self.postCalibrationMeasurement.run( 

641 measCat=sourceCat, 

642 exposure=exposure, 

643 exposureId=idGenerator.catalog_id, 

644 ) 

645 

646 if self.config.doComputeSummaryStats: 

647 summary = self.computeSummaryStats.run(exposure=exposure, 

648 sources=sourceCat, 

649 background=background) 

650 exposure.getInfo().setSummaryStats(summary) 

651 

652 frame = getDebugFrame(self._display, "calibrate") 

653 if frame: 

654 displayAstrometry( 

655 sourceCat=sourceCat, 

656 exposure=exposure, 

657 matches=astromMatches, 

658 frame=frame, 

659 pause=False, 

660 ) 

661 

662 return pipeBase.Struct( 

663 sourceCat=sourceCat, 

664 astromMatches=astromMatches, 

665 matchMeta=matchMeta, 

666 outputExposure=exposure, 

667 outputCat=sourceCat, 

668 outputBackground=background, 

669 ) 

670 

671 def setMetadata(self, exposure, photoRes=None): 

672 """Set task and exposure metadata. 

673 

674 Logs a warning continues if needed data is missing. 

675 

676 Parameters 

677 ---------- 

678 exposure : `lsst.afw.image.ExposureF` 

679 Exposure to set metadata on. 

680 photoRes : `lsst.pipe.base.Struct`, optional 

681 Result of running photoCal task. 

682 """ 

683 if photoRes is None: 

684 return 

685 

686 metadata = exposure.getMetadata() 

687 

688 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

689 try: 

690 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

691 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

692 except Exception: 

693 self.log.warning("Could not set normalized MAGZERO in header: no " 

694 "exposure time") 

695 magZero = math.nan 

696 

697 try: 

698 metadata.set('MAGZERO', magZero) 

699 metadata.set('MAGZERO_RMS', photoRes.sigma) 

700 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

701 metadata.set('COLORTERM1', 0.0) 

702 metadata.set('COLORTERM2', 0.0) 

703 metadata.set('COLORTERM3', 0.0) 

704 except Exception as e: 

705 self.log.warning("Could not set exposure metadata: %s", e) 

706 

707 def copyIcSourceFields(self, icSourceCat, sourceCat): 

708 """Match sources in an icSourceCat and a sourceCat and copy fields. 

709 

710 The fields copied are those specified by 

711 ``config.icSourceFieldsToCopy``. 

712 

713 Parameters 

714 ---------- 

715 icSourceCat : `lsst.afw.table.SourceCatalog` 

716 Catalog from which to copy fields. 

717 sourceCat : `lsst.afw.table.SourceCatalog` 

718 Catalog to which to copy fields. 

719 

720 Raises 

721 ------ 

722 RuntimeError 

723 Raised if any of the following occur: 

724 - icSourceSchema and icSourceKeys are not specified. 

725 - icSourceCat and sourceCat are not specified. 

726 - icSourceFieldsToCopy is empty. 

727 """ 

728 if self.schemaMapper is None: 

729 raise RuntimeError("To copy icSource fields you must specify " 

730 "icSourceSchema and icSourceKeys when " 

731 "constructing this task") 

732 if icSourceCat is None or sourceCat is None: 

733 raise RuntimeError("icSourceCat and sourceCat must both be " 

734 "specified") 

735 if len(self.config.icSourceFieldsToCopy) == 0: 

736 self.log.warning("copyIcSourceFields doing nothing because " 

737 "icSourceFieldsToCopy is empty") 

738 return 

739 

740 mc = afwTable.MatchControl() 

741 mc.findOnlyClosest = False # return all matched objects 

742 matches = afwTable.matchXy(icSourceCat, sourceCat, 

743 self.config.matchRadiusPix, mc) 

744 if self.config.doDeblend: 

745 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

746 # if deblended, keep children 

747 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

748 

749 # Because we had to allow multiple matches to handle parents, we now 

750 # need to prune to the best matches 

751 # closest matches as a dict of icSourceCat source ID: 

752 # (icSourceCat source, sourceCat source, distance in pixels) 

753 bestMatches = {} 

754 for m0, m1, d in matches: 

755 id0 = m0.getId() 

756 match = bestMatches.get(id0) 

757 if match is None or d <= match[2]: 

758 bestMatches[id0] = (m0, m1, d) 

759 matches = list(bestMatches.values()) 

760 

761 # Check that no sourceCat sources are listed twice (we already know 

762 # that each match has a unique icSourceCat source ID, due to using 

763 # that ID as the key in bestMatches) 

764 numMatches = len(matches) 

765 numUniqueSources = len(set(m[1].getId() for m in matches)) 

766 if numUniqueSources != numMatches: 

767 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

768 "sources", numMatches, numUniqueSources) 

769 

770 self.log.info("Copying flags from icSourceCat to sourceCat for " 

771 "%d sources", numMatches) 

772 

773 # For each match: set the calibSourceKey flag and copy the desired 

774 # fields 

775 for icSrc, src, d in matches: 

776 src.setFlag(self.calibSourceKey, True) 

777 # src.assign copies the footprint from icSrc, which we don't want 

778 # (DM-407) 

779 # so set icSrc's footprint to src's footprint before src.assign, 

780 # then restore it 

781 icSrcFootprint = icSrc.getFootprint() 

782 try: 

783 icSrc.setFootprint(src.getFootprint()) 

784 src.assign(icSrc, self.schemaMapper) 

785 finally: 

786 icSrc.setFootprint(icSrcFootprint)