Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

284 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-18 10:37 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import warnings 

26import numpy as np 

27 

28from lsstDebug import getDebugFrame 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.pipe.base.connectionTypes as cT 

32import lsst.afw.table as afwTable 

33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

34from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

35import lsst.daf.base as dafBase 

36from lsst.afw.math import BackgroundList 

37from lsst.afw.table import SourceTable 

38from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

39from lsst.meas.base import (SingleFrameMeasurementTask, 

40 ApplyApCorrTask, 

41 CatalogCalculationTask, 

42 IdGenerator, 

43 DetectorVisitIdGeneratorConfig) 

44from lsst.meas.deblender import SourceDeblendTask 

45from lsst.utils.timer import timeMethod 

46from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

47from .photoCal import PhotoCalTask 

48from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

49 

50 

51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

52 defaultTemplates={}): 

53 

54 icSourceSchema = cT.InitInput( 

55 doc="Schema produced by characterize image task, used to initialize this task", 

56 name="icSrc_schema", 

57 storageClass="SourceCatalog", 

58 ) 

59 

60 outputSchema = cT.InitOutput( 

61 doc="Schema after CalibrateTask has been initialized", 

62 name="src_schema", 

63 storageClass="SourceCatalog", 

64 ) 

65 

66 exposure = cT.Input( 

67 doc="Input image to calibrate", 

68 name="icExp", 

69 storageClass="ExposureF", 

70 dimensions=("instrument", "visit", "detector"), 

71 ) 

72 

73 background = cT.Input( 

74 doc="Backgrounds determined by characterize task", 

75 name="icExpBackground", 

76 storageClass="Background", 

77 dimensions=("instrument", "visit", "detector"), 

78 ) 

79 

80 icSourceCat = cT.Input( 

81 doc="Source catalog created by characterize task", 

82 name="icSrc", 

83 storageClass="SourceCatalog", 

84 dimensions=("instrument", "visit", "detector"), 

85 ) 

86 

87 astromRefCat = cT.PrerequisiteInput( 

88 doc="Reference catalog to use for astrometry", 

89 name="gaia_dr2_20200414", 

90 storageClass="SimpleCatalog", 

91 dimensions=("skypix",), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 photoRefCat = cT.PrerequisiteInput( 

97 doc="Reference catalog to use for photometric calibration", 

98 name="ps1_pv3_3pi_20170110", 

99 storageClass="SimpleCatalog", 

100 dimensions=("skypix",), 

101 deferLoad=True, 

102 multiple=True 

103 ) 

104 

105 outputExposure = cT.Output( 

106 doc="Exposure after running calibration task", 

107 name="calexp", 

108 storageClass="ExposureF", 

109 dimensions=("instrument", "visit", "detector"), 

110 ) 

111 

112 outputCat = cT.Output( 

113 doc="Source catalog produced in calibrate task", 

114 name="src", 

115 storageClass="SourceCatalog", 

116 dimensions=("instrument", "visit", "detector"), 

117 ) 

118 

119 outputBackground = cT.Output( 

120 doc="Background models estimated in calibration task", 

121 name="calexpBackground", 

122 storageClass="Background", 

123 dimensions=("instrument", "visit", "detector"), 

124 ) 

125 

126 matches = cT.Output( 

127 doc="Source/refObj matches from the astrometry solver", 

128 name="srcMatch", 

129 storageClass="Catalog", 

130 dimensions=("instrument", "visit", "detector"), 

131 ) 

132 

133 matchesDenormalized = cT.Output( 

134 doc="Denormalized matches from astrometry solver", 

135 name="srcMatchFull", 

136 storageClass="Catalog", 

137 dimensions=("instrument", "visit", "detector"), 

138 ) 

139 

140 def __init__(self, *, config=None): 

141 super().__init__(config=config) 

142 

143 if config.doAstrometry is False: 

144 self.prerequisiteInputs.remove("astromRefCat") 

145 if config.doPhotoCal is False: 

146 self.prerequisiteInputs.remove("photoRefCat") 

147 

148 if config.doWriteMatches is False or config.doAstrometry is False: 

149 self.outputs.remove("matches") 

150 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

151 self.outputs.remove("matchesDenormalized") 

152 

153 

154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

155 """Config for CalibrateTask.""" 

156 

157 doWrite = pexConfig.Field( 

158 dtype=bool, 

159 default=True, 

160 doc="Save calibration results?", 

161 ) 

162 doWriteHeavyFootprintsInSources = pexConfig.Field( 

163 dtype=bool, 

164 default=True, 

165 doc="Include HeavyFootprint data in source table? If false then heavy " 

166 "footprints are saved as normal footprints, which saves some space" 

167 ) 

168 doWriteMatches = pexConfig.Field( 

169 dtype=bool, 

170 default=True, 

171 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

172 ) 

173 doWriteMatchesDenormalized = pexConfig.Field( 

174 dtype=bool, 

175 default=False, 

176 doc=("Write reference matches in denormalized format? " 

177 "This format uses more disk space, but is more convenient to " 

178 "read. Ignored if doWriteMatches=False or doWrite=False."), 

179 ) 

180 doAstrometry = pexConfig.Field( 

181 dtype=bool, 

182 default=True, 

183 doc="Perform astrometric calibration?", 

184 ) 

185 astromRefObjLoader = pexConfig.ConfigField( 

186 dtype=LoadReferenceObjectsConfig, 

187 doc="reference object loader for astrometric calibration", 

188 ) 

189 photoRefObjLoader = pexConfig.ConfigField( 

190 dtype=LoadReferenceObjectsConfig, 

191 doc="reference object loader for photometric calibration", 

192 ) 

193 astrometry = pexConfig.ConfigurableField( 

194 target=AstrometryTask, 

195 doc="Perform astrometric calibration to refine the WCS", 

196 ) 

197 requireAstrometry = pexConfig.Field( 

198 dtype=bool, 

199 default=True, 

200 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

201 "false."), 

202 ) 

203 doPhotoCal = pexConfig.Field( 

204 dtype=bool, 

205 default=True, 

206 doc="Perform phometric calibration?", 

207 ) 

208 requirePhotoCal = pexConfig.Field( 

209 dtype=bool, 

210 default=True, 

211 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

212 "false."), 

213 ) 

214 photoCal = pexConfig.ConfigurableField( 

215 target=PhotoCalTask, 

216 doc="Perform photometric calibration", 

217 ) 

218 icSourceFieldsToCopy = pexConfig.ListField( 

219 dtype=str, 

220 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

221 doc=("Fields to copy from the icSource catalog to the output catalog " 

222 "for matching sources Any missing fields will trigger a " 

223 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

224 ) 

225 matchRadiusPix = pexConfig.Field( 

226 dtype=float, 

227 default=3, 

228 doc=("Match radius for matching icSourceCat objects to sourceCat " 

229 "objects (pixels)"), 

230 ) 

231 checkUnitsParseStrict = pexConfig.Field( 

232 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

233 "'warn' or 'silent'"), 

234 dtype=str, 

235 default="raise", 

236 ) 

237 detection = pexConfig.ConfigurableField( 

238 target=SourceDetectionTask, 

239 doc="Detect sources" 

240 ) 

241 doDeblend = pexConfig.Field( 

242 dtype=bool, 

243 default=True, 

244 doc="Run deblender input exposure" 

245 ) 

246 deblend = pexConfig.ConfigurableField( 

247 target=SourceDeblendTask, 

248 doc="Split blended sources into their components" 

249 ) 

250 doSkySources = pexConfig.Field( 

251 dtype=bool, 

252 default=True, 

253 doc="Generate sky sources?", 

254 ) 

255 skySources = pexConfig.ConfigurableField( 

256 target=SkyObjectsTask, 

257 doc="Generate sky sources", 

258 ) 

259 measurement = pexConfig.ConfigurableField( 

260 target=SingleFrameMeasurementTask, 

261 doc="Measure sources" 

262 ) 

263 postCalibrationMeasurement = pexConfig.ConfigurableField( 

264 target=SingleFrameMeasurementTask, 

265 doc="Second round of measurement for plugins that need to be run after photocal" 

266 ) 

267 setPrimaryFlags = pexConfig.ConfigurableField( 

268 target=SetPrimaryFlagsTask, 

269 doc=("Set flags for primary source classification in single frame " 

270 "processing. True if sources are not sky sources and not a parent.") 

271 ) 

272 doApCorr = pexConfig.Field( 

273 dtype=bool, 

274 default=True, 

275 doc="Run subtask to apply aperture correction" 

276 ) 

277 applyApCorr = pexConfig.ConfigurableField( 

278 target=ApplyApCorrTask, 

279 doc="Subtask to apply aperture corrections" 

280 ) 

281 # If doApCorr is False, and the exposure does not have apcorrections 

282 # already applied, the active plugins in catalogCalculation almost 

283 # certainly should not contain the characterization plugin 

284 catalogCalculation = pexConfig.ConfigurableField( 

285 target=CatalogCalculationTask, 

286 doc="Subtask to run catalogCalculation plugins on catalog" 

287 ) 

288 doComputeSummaryStats = pexConfig.Field( 

289 dtype=bool, 

290 default=True, 

291 doc="Run subtask to measure exposure summary statistics?" 

292 ) 

293 computeSummaryStats = pexConfig.ConfigurableField( 

294 target=ComputeExposureSummaryStatsTask, 

295 doc="Subtask to run computeSummaryStats on exposure" 

296 ) 

297 doWriteExposure = pexConfig.Field( 

298 dtype=bool, 

299 default=True, 

300 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

301 "normal calexp but as a fakes_calexp." 

302 ) 

303 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

304 

305 def setDefaults(self): 

306 super().setDefaults() 

307 self.detection.doTempLocalBackground = False 

308 self.deblend.maxFootprintSize = 2000 

309 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

310 self.postCalibrationMeasurement.doReplaceWithNoise = False 

311 for key in self.postCalibrationMeasurement.slots: 

312 setattr(self.postCalibrationMeasurement.slots, key, None) 

313 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

314 # The photoRefCat connection is the name to use for the colorterms. 

315 self.photoCal.photoCatName = self.connections.photoRefCat 

316 

317 

318class CalibrateTask(pipeBase.PipelineTask): 

319 """Calibrate an exposure: measure sources and perform astrometric and 

320 photometric calibration. 

321 

322 Given an exposure with a good PSF model and aperture correction map(e.g. as 

323 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

324 perform the following operations: 

325 - Run detection and measurement 

326 - Run astrometry subtask to fit an improved WCS 

327 - Run photoCal subtask to fit the exposure's photometric zero-point 

328 

329 Parameters 

330 ---------- 

331 butler : `None` 

332 Compatibility parameter. Should always be `None`. 

333 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

334 Unused in gen3: must be `None`. 

335 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

336 Unused in gen3: must be `None`. 

337 icSourceSchema : `lsst.afw.table.Schema`, optional 

338 Schema for the icSource catalog. 

339 initInputs : `dict`, optional 

340 Dictionary that can contain a key ``icSourceSchema`` containing the 

341 input schema. If present will override the value of ``icSourceSchema``. 

342 

343 Raises 

344 ------ 

345 RuntimeError 

346 Raised if any of the following occur: 

347 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

348 - PipelineTask form of this task is initialized with reference object 

349 loaders. 

350 

351 Notes 

352 ----- 

353 Quantities set in exposure Metadata: 

354 

355 MAGZERO_RMS 

356 MAGZERO's RMS == sigma reported by photoCal task 

357 MAGZERO_NOBJ 

358 Number of stars used == ngood reported by photoCal task 

359 COLORTERM1 

360 ?? (always 0.0) 

361 COLORTERM2 

362 ?? (always 0.0) 

363 COLORTERM3 

364 ?? (always 0.0) 

365 

366 Debugging: 

367 CalibrateTask has a debug dictionary containing one key: 

368 

369 calibrate 

370 frame (an int; <= 0 to not display) in which to display the exposure, 

371 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

372 the meaning of the various symbols. 

373 """ 

374 

375 ConfigClass = CalibrateConfig 

376 _DefaultName = "calibrate" 

377 

378 def __init__(self, butler=None, astromRefObjLoader=None, 

379 photoRefObjLoader=None, icSourceSchema=None, 

380 initInputs=None, **kwargs): 

381 super().__init__(**kwargs) 

382 

383 if butler is not None: 

384 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

385 category=FutureWarning, stacklevel=2) 

386 butler = None 

387 

388 if initInputs is not None: 

389 icSourceSchema = initInputs['icSourceSchema'].schema 

390 

391 if icSourceSchema is not None: 

392 # use a schema mapper to avoid copying each field separately 

393 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

394 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

395 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

396 

397 # Add fields to copy from an icSource catalog 

398 # and a field to indicate that the source matched a source in that 

399 # catalog. If any fields are missing then raise an exception, but 

400 # first find all missing fields in order to make the error message 

401 # more useful. 

402 self.calibSourceKey = self.schemaMapper.addOutputField( 

403 afwTable.Field["Flag"]("calib_detected", 

404 "Source was detected as an icSource")) 

405 missingFieldNames = [] 

406 for fieldName in self.config.icSourceFieldsToCopy: 

407 try: 

408 schemaItem = icSourceSchema.find(fieldName) 

409 except Exception: 

410 missingFieldNames.append(fieldName) 

411 else: 

412 # field found; if addMapping fails then raise an exception 

413 self.schemaMapper.addMapping(schemaItem.getKey()) 

414 

415 if missingFieldNames: 

416 raise RuntimeError("isSourceCat is missing fields {} " 

417 "specified in icSourceFieldsToCopy" 

418 .format(missingFieldNames)) 

419 

420 # produce a temporary schema to pass to the subtasks; finalize it 

421 # later 

422 self.schema = self.schemaMapper.editOutputSchema() 

423 else: 

424 self.schemaMapper = None 

425 self.schema = afwTable.SourceTable.makeMinimalSchema() 

426 self.makeSubtask('detection', schema=self.schema) 

427 

428 self.algMetadata = dafBase.PropertyList() 

429 

430 if self.config.doDeblend: 

431 self.makeSubtask("deblend", schema=self.schema) 

432 if self.config.doSkySources: 

433 self.makeSubtask("skySources") 

434 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

435 self.makeSubtask('measurement', schema=self.schema, 

436 algMetadata=self.algMetadata) 

437 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

438 algMetadata=self.algMetadata) 

439 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

440 if self.config.doApCorr: 

441 self.makeSubtask('applyApCorr', schema=self.schema) 

442 self.makeSubtask('catalogCalculation', schema=self.schema) 

443 

444 if self.config.doAstrometry: 

445 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

446 schema=self.schema) 

447 if self.config.doPhotoCal: 

448 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

449 schema=self.schema) 

450 if self.config.doComputeSummaryStats: 

451 self.makeSubtask('computeSummaryStats') 

452 

453 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

454 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

455 "reference object loaders.") 

456 

457 if self.schemaMapper is not None: 

458 # finalize the schema 

459 self.schema = self.schemaMapper.getOutputSchema() 

460 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

461 

462 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

463 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

464 self.outputSchema = sourceCatSchema 

465 

466 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

467 inputs = butlerQC.get(inputRefs) 

468 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

469 

470 if self.config.doAstrometry: 

471 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

472 for ref in inputRefs.astromRefCat], 

473 refCats=inputs.pop('astromRefCat'), 

474 name=self.config.connections.astromRefCat, 

475 config=self.config.astromRefObjLoader, log=self.log) 

476 self.astrometry.setRefObjLoader(refObjLoader) 

477 

478 if self.config.doPhotoCal: 

479 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

480 for ref in inputRefs.photoRefCat], 

481 refCats=inputs.pop('photoRefCat'), 

482 name=self.config.connections.photoRefCat, 

483 config=self.config.photoRefObjLoader, 

484 log=self.log) 

485 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

486 

487 outputs = self.run(**inputs) 

488 

489 if self.config.doWriteMatches and self.config.doAstrometry: 

490 if outputs.astromMatches is not None: 

491 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

492 normalizedMatches.table.setMetadata(outputs.matchMeta) 

493 if self.config.doWriteMatchesDenormalized: 

494 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

495 outputs.matchesDenormalized = denormMatches 

496 outputs.matches = normalizedMatches 

497 else: 

498 del outputRefs.matches 

499 if self.config.doWriteMatchesDenormalized: 

500 del outputRefs.matchesDenormalized 

501 butlerQC.put(outputs, outputRefs) 

502 

503 @timeMethod 

504 def run(self, exposure, exposureIdInfo=None, background=None, 

505 icSourceCat=None, idGenerator=None): 

506 """Calibrate an exposure. 

507 

508 Parameters 

509 ---------- 

510 exposure : `lsst.afw.image.ExposureF` 

511 Exposure to calibrate. 

512 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

513 Exposure ID info. Deprecated in favor of ``idGenerator``, and 

514 ignored if that is provided. 

515 background : `lsst.afw.math.BackgroundList`, optional 

516 Initial model of background already subtracted from exposure. 

517 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

518 SourceCatalog from CharacterizeImageTask from which we can copy 

519 some fields. 

520 idGenerator : `lsst.meas.base.IdGenerator`, optional 

521 Object that generates source IDs and provides RNG seeds. 

522 

523 Returns 

524 ------- 

525 result : `lsst.pipe.base.Struct` 

526 Results as a struct with attributes: 

527 

528 ``exposure`` 

529 Characterized exposure (`lsst.afw.image.ExposureF`). 

530 ``sourceCat`` 

531 Detected sources (`lsst.afw.table.SourceCatalog`). 

532 ``outputBackground`` 

533 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

534 ``astromMatches`` 

535 List of source/ref matches from astrometry solver. 

536 ``matchMeta`` 

537 Metadata from astrometry matches. 

538 ``outputExposure`` 

539 Another reference to ``exposure`` for compatibility. 

540 ``outputCat`` 

541 Another reference to ``sourceCat`` for compatibility. 

542 """ 

543 # detect, deblend and measure sources 

544 if idGenerator is None: 

545 if exposureIdInfo is not None: 

546 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

547 else: 

548 idGenerator = IdGenerator() 

549 

550 if background is None: 

551 background = BackgroundList() 

552 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

553 table.setMetadata(self.algMetadata) 

554 

555 detRes = self.detection.run(table=table, exposure=exposure, 

556 doSmooth=True) 

557 sourceCat = detRes.sources 

558 if detRes.background: 

559 for bg in detRes.background: 

560 background.append(bg) 

561 if self.config.doSkySources: 

562 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

563 if skySourceFootprints: 

564 for foot in skySourceFootprints: 

565 s = sourceCat.addNew() 

566 s.setFootprint(foot) 

567 s.set(self.skySourceKey, True) 

568 if self.config.doDeblend: 

569 self.deblend.run(exposure=exposure, sources=sourceCat) 

570 self.measurement.run( 

571 measCat=sourceCat, 

572 exposure=exposure, 

573 exposureId=idGenerator.catalog_id, 

574 ) 

575 if self.config.doApCorr: 

576 apCorrMap = exposure.getInfo().getApCorrMap() 

577 if apCorrMap is None: 

578 self.log.warning("Image does not have valid aperture correction map for %r; " 

579 "skipping aperture correction", idGenerator) 

580 else: 

581 self.applyApCorr.run( 

582 catalog=sourceCat, 

583 apCorrMap=apCorrMap, 

584 ) 

585 self.catalogCalculation.run(sourceCat) 

586 

587 self.setPrimaryFlags.run(sourceCat) 

588 

589 if icSourceCat is not None and \ 

590 len(self.config.icSourceFieldsToCopy) > 0: 

591 self.copyIcSourceFields(icSourceCat=icSourceCat, 

592 sourceCat=sourceCat) 

593 

594 # TODO DM-11568: this contiguous check-and-copy could go away if we 

595 # reserve enough space during SourceDetection and/or SourceDeblend. 

596 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

597 # contiguity now, so views are preserved from here on. 

598 if not sourceCat.isContiguous(): 

599 sourceCat = sourceCat.copy(deep=True) 

600 

601 # perform astrometry calibration: 

602 # fit an improved WCS and update the exposure's WCS in place 

603 astromMatches = None 

604 matchMeta = None 

605 if self.config.doAstrometry: 

606 astromRes = self.astrometry.run( 

607 exposure=exposure, 

608 sourceCat=sourceCat, 

609 ) 

610 astromMatches = astromRes.matches 

611 matchMeta = astromRes.matchMeta 

612 if exposure.getWcs() is None: 

613 if self.config.requireAstrometry: 

614 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

615 "is True.") 

616 else: 

617 self.log.warning("Unable to perform astrometric calibration for %r but " 

618 "requireAstrometry is False: attempting to proceed...", 

619 idGenerator) 

620 

621 # compute photometric calibration 

622 if self.config.doPhotoCal: 

623 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

624 if self.config.requirePhotoCal: 

625 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

626 "photoCal, but requirePhotoCal is True.") 

627 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

628 "is False, so skipping photometric calibration and setting photoCalib " 

629 "to None. Attempting to proceed...", idGenerator) 

630 exposure.setPhotoCalib(None) 

631 self.setMetadata(exposure=exposure, photoRes=None) 

632 else: 

633 try: 

634 photoRes = self.photoCal.run( 

635 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

636 ) 

637 exposure.setPhotoCalib(photoRes.photoCalib) 

638 # TODO: reword this to phrase it in terms of the 

639 # calibration factor? 

640 self.log.info("Photometric zero-point: %f", 

641 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

642 self.setMetadata(exposure=exposure, photoRes=photoRes) 

643 except Exception as e: 

644 if self.config.requirePhotoCal: 

645 raise 

646 self.log.warning("Unable to perform photometric calibration " 

647 "(%s): attempting to proceed", e) 

648 self.setMetadata(exposure=exposure, photoRes=None) 

649 

650 self.postCalibrationMeasurement.run( 

651 measCat=sourceCat, 

652 exposure=exposure, 

653 exposureId=idGenerator.catalog_id, 

654 ) 

655 

656 if self.config.doComputeSummaryStats: 

657 summary = self.computeSummaryStats.run(exposure=exposure, 

658 sources=sourceCat, 

659 background=background) 

660 exposure.getInfo().setSummaryStats(summary) 

661 

662 frame = getDebugFrame(self._display, "calibrate") 

663 if frame: 

664 displayAstrometry( 

665 sourceCat=sourceCat, 

666 exposure=exposure, 

667 matches=astromMatches, 

668 frame=frame, 

669 pause=False, 

670 ) 

671 

672 return pipeBase.Struct( 

673 sourceCat=sourceCat, 

674 astromMatches=astromMatches, 

675 matchMeta=matchMeta, 

676 outputExposure=exposure, 

677 outputCat=sourceCat, 

678 outputBackground=background, 

679 ) 

680 

681 def setMetadata(self, exposure, photoRes=None): 

682 """Set task and exposure metadata. 

683 

684 Logs a warning continues if needed data is missing. 

685 

686 Parameters 

687 ---------- 

688 exposure : `lsst.afw.image.ExposureF` 

689 Exposure to set metadata on. 

690 photoRes : `lsst.pipe.base.Struct`, optional 

691 Result of running photoCal task. 

692 """ 

693 if photoRes is None: 

694 return 

695 

696 metadata = exposure.getMetadata() 

697 

698 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

699 try: 

700 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

701 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

702 except Exception: 

703 self.log.warning("Could not set normalized MAGZERO in header: no " 

704 "exposure time") 

705 magZero = math.nan 

706 

707 try: 

708 metadata.set('MAGZERO', magZero) 

709 metadata.set('MAGZERO_RMS', photoRes.sigma) 

710 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

711 metadata.set('COLORTERM1', 0.0) 

712 metadata.set('COLORTERM2', 0.0) 

713 metadata.set('COLORTERM3', 0.0) 

714 except Exception as e: 

715 self.log.warning("Could not set exposure metadata: %s", e) 

716 

717 def copyIcSourceFields(self, icSourceCat, sourceCat): 

718 """Match sources in an icSourceCat and a sourceCat and copy fields. 

719 

720 The fields copied are those specified by 

721 ``config.icSourceFieldsToCopy``. 

722 

723 Parameters 

724 ---------- 

725 icSourceCat : `lsst.afw.table.SourceCatalog` 

726 Catalog from which to copy fields. 

727 sourceCat : `lsst.afw.table.SourceCatalog` 

728 Catalog to which to copy fields. 

729 

730 Raises 

731 ------ 

732 RuntimeError 

733 Raised if any of the following occur: 

734 - icSourceSchema and icSourceKeys are not specified. 

735 - icSourceCat and sourceCat are not specified. 

736 - icSourceFieldsToCopy is empty. 

737 """ 

738 if self.schemaMapper is None: 

739 raise RuntimeError("To copy icSource fields you must specify " 

740 "icSourceSchema and icSourceKeys when " 

741 "constructing this task") 

742 if icSourceCat is None or sourceCat is None: 

743 raise RuntimeError("icSourceCat and sourceCat must both be " 

744 "specified") 

745 if len(self.config.icSourceFieldsToCopy) == 0: 

746 self.log.warning("copyIcSourceFields doing nothing because " 

747 "icSourceFieldsToCopy is empty") 

748 return 

749 

750 mc = afwTable.MatchControl() 

751 mc.findOnlyClosest = False # return all matched objects 

752 matches = afwTable.matchXy(icSourceCat, sourceCat, 

753 self.config.matchRadiusPix, mc) 

754 if self.config.doDeblend: 

755 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

756 # if deblended, keep children 

757 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

758 

759 # Because we had to allow multiple matches to handle parents, we now 

760 # need to prune to the best matches 

761 # closest matches as a dict of icSourceCat source ID: 

762 # (icSourceCat source, sourceCat source, distance in pixels) 

763 bestMatches = {} 

764 for m0, m1, d in matches: 

765 id0 = m0.getId() 

766 match = bestMatches.get(id0) 

767 if match is None or d <= match[2]: 

768 bestMatches[id0] = (m0, m1, d) 

769 matches = list(bestMatches.values()) 

770 

771 # Check that no sourceCat sources are listed twice (we already know 

772 # that each match has a unique icSourceCat source ID, due to using 

773 # that ID as the key in bestMatches) 

774 numMatches = len(matches) 

775 numUniqueSources = len(set(m[1].getId() for m in matches)) 

776 if numUniqueSources != numMatches: 

777 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

778 "sources", numMatches, numUniqueSources) 

779 

780 self.log.info("Copying flags from icSourceCat to sourceCat for " 

781 "%d sources", numMatches) 

782 

783 # For each match: set the calibSourceKey flag and copy the desired 

784 # fields 

785 for icSrc, src, d in matches: 

786 src.setFlag(self.calibSourceKey, True) 

787 # src.assign copies the footprint from icSrc, which we don't want 

788 # (DM-407) 

789 # so set icSrc's footprint to src's footprint before src.assign, 

790 # then restore it 

791 icSrcFootprint = icSrc.getFootprint() 

792 try: 

793 icSrc.setFootprint(src.getFootprint()) 

794 src.assign(icSrc, self.schemaMapper) 

795 finally: 

796 icSrc.setFootprint(icSrcFootprint)