Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

281 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-04 13:31 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import numpy as np 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

33from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

34import lsst.daf.base as dafBase 

35from lsst.afw.math import BackgroundList 

36from lsst.afw.table import SourceTable 

37from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

38from lsst.meas.base import (SingleFrameMeasurementTask, 

39 ApplyApCorrTask, 

40 CatalogCalculationTask, 

41 IdGenerator, 

42 DetectorVisitIdGeneratorConfig) 

43from lsst.meas.deblender import SourceDeblendTask 

44from lsst.utils.timer import timeMethod 

45from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

46from .photoCal import PhotoCalTask 

47from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="gaia_dr3_20230707", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="ps1_pv3_3pi_20170110", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask.""" 

155 

156 doWrite = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Save calibration results?", 

160 ) 

161 doWriteHeavyFootprintsInSources = pexConfig.Field( 

162 dtype=bool, 

163 default=True, 

164 doc="Include HeavyFootprint data in source table? If false then heavy " 

165 "footprints are saved as normal footprints, which saves some space" 

166 ) 

167 doWriteMatches = pexConfig.Field( 

168 dtype=bool, 

169 default=True, 

170 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

171 ) 

172 doWriteMatchesDenormalized = pexConfig.Field( 

173 dtype=bool, 

174 default=True, 

175 doc=("Write reference matches in denormalized format? " 

176 "This format uses more disk space, but is more convenient to " 

177 "read for debugging. Ignored if doWriteMatches=False or doWrite=False."), 

178 ) 

179 doAstrometry = pexConfig.Field( 

180 dtype=bool, 

181 default=True, 

182 doc="Perform astrometric calibration?", 

183 ) 

184 astromRefObjLoader = pexConfig.ConfigField( 

185 dtype=LoadReferenceObjectsConfig, 

186 doc="reference object loader for astrometric calibration", 

187 ) 

188 photoRefObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 doc="reference object loader for photometric calibration", 

191 ) 

192 astrometry = pexConfig.ConfigurableField( 

193 target=AstrometryTask, 

194 doc="Perform astrometric calibration to refine the WCS", 

195 ) 

196 requireAstrometry = pexConfig.Field( 

197 dtype=bool, 

198 default=True, 

199 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

200 "false."), 

201 ) 

202 doPhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc="Perform phometric calibration?", 

206 ) 

207 requirePhotoCal = pexConfig.Field( 

208 dtype=bool, 

209 default=True, 

210 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

211 "false."), 

212 ) 

213 photoCal = pexConfig.ConfigurableField( 

214 target=PhotoCalTask, 

215 doc="Perform photometric calibration", 

216 ) 

217 icSourceFieldsToCopy = pexConfig.ListField( 

218 dtype=str, 

219 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

220 doc=("Fields to copy from the icSource catalog to the output catalog " 

221 "for matching sources Any missing fields will trigger a " 

222 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

223 ) 

224 matchRadiusPix = pexConfig.Field( 

225 dtype=float, 

226 default=3, 

227 doc=("Match radius for matching icSourceCat objects to sourceCat " 

228 "objects (pixels)"), 

229 ) 

230 checkUnitsParseStrict = pexConfig.Field( 

231 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

232 "'warn' or 'silent'"), 

233 dtype=str, 

234 default="raise", 

235 ) 

236 detection = pexConfig.ConfigurableField( 

237 target=SourceDetectionTask, 

238 doc="Detect sources" 

239 ) 

240 doDeblend = pexConfig.Field( 

241 dtype=bool, 

242 default=True, 

243 doc="Run deblender input exposure" 

244 ) 

245 deblend = pexConfig.ConfigurableField( 

246 target=SourceDeblendTask, 

247 doc="Split blended sources into their components" 

248 ) 

249 doSkySources = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Generate sky sources?", 

253 ) 

254 skySources = pexConfig.ConfigurableField( 

255 target=SkyObjectsTask, 

256 doc="Generate sky sources", 

257 ) 

258 measurement = pexConfig.ConfigurableField( 

259 target=SingleFrameMeasurementTask, 

260 doc="Measure sources" 

261 ) 

262 postCalibrationMeasurement = pexConfig.ConfigurableField( 

263 target=SingleFrameMeasurementTask, 

264 doc="Second round of measurement for plugins that need to be run after photocal" 

265 ) 

266 setPrimaryFlags = pexConfig.ConfigurableField( 

267 target=SetPrimaryFlagsTask, 

268 doc=("Set flags for primary source classification in single frame " 

269 "processing. True if sources are not sky sources and not a parent.") 

270 ) 

271 doApCorr = pexConfig.Field( 

272 dtype=bool, 

273 default=True, 

274 doc="Run subtask to apply aperture correction" 

275 ) 

276 applyApCorr = pexConfig.ConfigurableField( 

277 target=ApplyApCorrTask, 

278 doc="Subtask to apply aperture corrections" 

279 ) 

280 # If doApCorr is False, and the exposure does not have apcorrections 

281 # already applied, the active plugins in catalogCalculation almost 

282 # certainly should not contain the characterization plugin 

283 catalogCalculation = pexConfig.ConfigurableField( 

284 target=CatalogCalculationTask, 

285 doc="Subtask to run catalogCalculation plugins on catalog" 

286 ) 

287 doComputeSummaryStats = pexConfig.Field( 

288 dtype=bool, 

289 default=True, 

290 doc="Run subtask to measure exposure summary statistics?" 

291 ) 

292 computeSummaryStats = pexConfig.ConfigurableField( 

293 target=ComputeExposureSummaryStatsTask, 

294 doc="Subtask to run computeSummaryStats on exposure" 

295 ) 

296 doWriteExposure = pexConfig.Field( 

297 dtype=bool, 

298 default=True, 

299 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

300 "normal calexp but as a fakes_calexp." 

301 ) 

302 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

303 

304 def setDefaults(self): 

305 super().setDefaults() 

306 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

307 self.postCalibrationMeasurement.doReplaceWithNoise = False 

308 for key in self.postCalibrationMeasurement.slots: 

309 setattr(self.postCalibrationMeasurement.slots, key, None) 

310 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

311 # The photoRefCat connection is the name to use for the colorterms. 

312 self.photoCal.photoCatName = self.connections.photoRefCat 

313 

314 # Keep track of which footprints contain streaks 

315 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

316 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

317 

318 

319class CalibrateTask(pipeBase.PipelineTask): 

320 """Calibrate an exposure: measure sources and perform astrometric and 

321 photometric calibration. 

322 

323 Given an exposure with a good PSF model and aperture correction map(e.g. as 

324 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

325 perform the following operations: 

326 - Run detection and measurement 

327 - Run astrometry subtask to fit an improved WCS 

328 - Run photoCal subtask to fit the exposure's photometric zero-point 

329 

330 Parameters 

331 ---------- 

332 butler : `None` 

333 Compatibility parameter. Should always be `None`. 

334 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

335 Unused in gen3: must be `None`. 

336 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

337 Unused in gen3: must be `None`. 

338 icSourceSchema : `lsst.afw.table.Schema`, optional 

339 Schema for the icSource catalog. 

340 initInputs : `dict`, optional 

341 Dictionary that can contain a key ``icSourceSchema`` containing the 

342 input schema. If present will override the value of ``icSourceSchema``. 

343 

344 Raises 

345 ------ 

346 RuntimeError 

347 Raised if any of the following occur: 

348 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

349 - PipelineTask form of this task is initialized with reference object 

350 loaders. 

351 

352 Notes 

353 ----- 

354 Quantities set in exposure Metadata: 

355 

356 MAGZERO_RMS 

357 MAGZERO's RMS == sigma reported by photoCal task 

358 MAGZERO_NOBJ 

359 Number of stars used == ngood reported by photoCal task 

360 COLORTERM1 

361 ?? (always 0.0) 

362 COLORTERM2 

363 ?? (always 0.0) 

364 COLORTERM3 

365 ?? (always 0.0) 

366 

367 Debugging: 

368 CalibrateTask has a debug dictionary containing one key: 

369 

370 calibrate 

371 frame (an int; <= 0 to not display) in which to display the exposure, 

372 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

373 the meaning of the various symbols. 

374 """ 

375 

376 ConfigClass = CalibrateConfig 

377 _DefaultName = "calibrate" 

378 

379 def __init__(self, astromRefObjLoader=None, 

380 photoRefObjLoader=None, icSourceSchema=None, 

381 initInputs=None, **kwargs): 

382 super().__init__(**kwargs) 

383 

384 if initInputs is not None: 

385 icSourceSchema = initInputs['icSourceSchema'].schema 

386 

387 if icSourceSchema is not None: 

388 # use a schema mapper to avoid copying each field separately 

389 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

390 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

391 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

392 

393 # Add fields to copy from an icSource catalog 

394 # and a field to indicate that the source matched a source in that 

395 # catalog. If any fields are missing then raise an exception, but 

396 # first find all missing fields in order to make the error message 

397 # more useful. 

398 self.calibSourceKey = self.schemaMapper.addOutputField( 

399 afwTable.Field["Flag"]("calib_detected", 

400 "Source was detected as an icSource")) 

401 missingFieldNames = [] 

402 for fieldName in self.config.icSourceFieldsToCopy: 

403 try: 

404 schemaItem = icSourceSchema.find(fieldName) 

405 except Exception: 

406 missingFieldNames.append(fieldName) 

407 else: 

408 # field found; if addMapping fails then raise an exception 

409 self.schemaMapper.addMapping(schemaItem.getKey()) 

410 

411 if missingFieldNames: 

412 raise RuntimeError("isSourceCat is missing fields {} " 

413 "specified in icSourceFieldsToCopy" 

414 .format(missingFieldNames)) 

415 

416 # produce a temporary schema to pass to the subtasks; finalize it 

417 # later 

418 self.schema = self.schemaMapper.editOutputSchema() 

419 else: 

420 self.schemaMapper = None 

421 self.schema = afwTable.SourceTable.makeMinimalSchema() 

422 afwTable.CoordKey.addErrorFields(self.schema) 

423 self.makeSubtask('detection', schema=self.schema) 

424 

425 self.algMetadata = dafBase.PropertyList() 

426 

427 if self.config.doDeblend: 

428 self.makeSubtask("deblend", schema=self.schema) 

429 if self.config.doSkySources: 

430 self.makeSubtask("skySources") 

431 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

432 self.makeSubtask('measurement', schema=self.schema, 

433 algMetadata=self.algMetadata) 

434 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

435 algMetadata=self.algMetadata) 

436 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

437 if self.config.doApCorr: 

438 self.makeSubtask('applyApCorr', schema=self.schema) 

439 self.makeSubtask('catalogCalculation', schema=self.schema) 

440 

441 if self.config.doAstrometry: 

442 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

443 schema=self.schema) 

444 if self.config.doPhotoCal: 

445 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

446 schema=self.schema) 

447 if self.config.doComputeSummaryStats: 

448 self.makeSubtask('computeSummaryStats') 

449 

450 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

451 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

452 "reference object loaders.") 

453 

454 if self.schemaMapper is not None: 

455 # finalize the schema 

456 self.schema = self.schemaMapper.getOutputSchema() 

457 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

458 

459 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

460 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

461 self.outputSchema = sourceCatSchema 

462 

463 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

464 inputs = butlerQC.get(inputRefs) 

465 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

466 

467 if self.config.doAstrometry: 

468 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

469 for ref in inputRefs.astromRefCat], 

470 refCats=inputs.pop('astromRefCat'), 

471 name=self.config.connections.astromRefCat, 

472 config=self.config.astromRefObjLoader, log=self.log) 

473 self.astrometry.setRefObjLoader(refObjLoader) 

474 

475 if self.config.doPhotoCal: 

476 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

477 for ref in inputRefs.photoRefCat], 

478 refCats=inputs.pop('photoRefCat'), 

479 name=self.config.connections.photoRefCat, 

480 config=self.config.photoRefObjLoader, 

481 log=self.log) 

482 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

483 

484 outputs = self.run(**inputs) 

485 

486 if self.config.doWriteMatches and self.config.doAstrometry: 

487 if outputs.astromMatches is not None: 

488 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

489 normalizedMatches.table.setMetadata(outputs.matchMeta) 

490 if self.config.doWriteMatchesDenormalized: 

491 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

492 outputs.matchesDenormalized = denormMatches 

493 outputs.matches = normalizedMatches 

494 else: 

495 del outputRefs.matches 

496 if self.config.doWriteMatchesDenormalized: 

497 del outputRefs.matchesDenormalized 

498 butlerQC.put(outputs, outputRefs) 

499 

500 @timeMethod 

501 def run(self, exposure, exposureIdInfo=None, background=None, 

502 icSourceCat=None, idGenerator=None): 

503 """Calibrate an exposure. 

504 

505 Parameters 

506 ---------- 

507 exposure : `lsst.afw.image.ExposureF` 

508 Exposure to calibrate. 

509 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

510 Exposure ID info. Deprecated in favor of ``idGenerator``, and 

511 ignored if that is provided. 

512 background : `lsst.afw.math.BackgroundList`, optional 

513 Initial model of background already subtracted from exposure. 

514 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

515 SourceCatalog from CharacterizeImageTask from which we can copy 

516 some fields. 

517 idGenerator : `lsst.meas.base.IdGenerator`, optional 

518 Object that generates source IDs and provides RNG seeds. 

519 

520 Returns 

521 ------- 

522 result : `lsst.pipe.base.Struct` 

523 Results as a struct with attributes: 

524 

525 ``exposure`` 

526 Characterized exposure (`lsst.afw.image.ExposureF`). 

527 ``sourceCat`` 

528 Detected sources (`lsst.afw.table.SourceCatalog`). 

529 ``outputBackground`` 

530 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

531 ``astromMatches`` 

532 List of source/ref matches from astrometry solver. 

533 ``matchMeta`` 

534 Metadata from astrometry matches. 

535 ``outputExposure`` 

536 Another reference to ``exposure`` for compatibility. 

537 ``outputCat`` 

538 Another reference to ``sourceCat`` for compatibility. 

539 """ 

540 # detect, deblend and measure sources 

541 if idGenerator is None: 

542 if exposureIdInfo is not None: 

543 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

544 else: 

545 idGenerator = IdGenerator() 

546 

547 if background is None: 

548 background = BackgroundList() 

549 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

550 table.setMetadata(self.algMetadata) 

551 

552 detRes = self.detection.run(table=table, exposure=exposure, 

553 doSmooth=True) 

554 sourceCat = detRes.sources 

555 if detRes.background: 

556 for bg in detRes.background: 

557 background.append(bg) 

558 if self.config.doSkySources: 

559 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

560 if skySourceFootprints: 

561 for foot in skySourceFootprints: 

562 s = sourceCat.addNew() 

563 s.setFootprint(foot) 

564 s.set(self.skySourceKey, True) 

565 if self.config.doDeblend: 

566 self.deblend.run(exposure=exposure, sources=sourceCat) 

567 self.measurement.run( 

568 measCat=sourceCat, 

569 exposure=exposure, 

570 exposureId=idGenerator.catalog_id, 

571 ) 

572 if self.config.doApCorr: 

573 apCorrMap = exposure.getInfo().getApCorrMap() 

574 if apCorrMap is None: 

575 self.log.warning("Image does not have valid aperture correction map for %r; " 

576 "skipping aperture correction", idGenerator) 

577 else: 

578 self.applyApCorr.run( 

579 catalog=sourceCat, 

580 apCorrMap=apCorrMap, 

581 ) 

582 self.catalogCalculation.run(sourceCat) 

583 

584 self.setPrimaryFlags.run(sourceCat) 

585 

586 if icSourceCat is not None and \ 

587 len(self.config.icSourceFieldsToCopy) > 0: 

588 self.copyIcSourceFields(icSourceCat=icSourceCat, 

589 sourceCat=sourceCat) 

590 

591 # TODO DM-11568: this contiguous check-and-copy could go away if we 

592 # reserve enough space during SourceDetection and/or SourceDeblend. 

593 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

594 # contiguity now, so views are preserved from here on. 

595 if not sourceCat.isContiguous(): 

596 sourceCat = sourceCat.copy(deep=True) 

597 

598 # perform astrometry calibration: 

599 # fit an improved WCS and update the exposure's WCS in place 

600 astromMatches = None 

601 matchMeta = None 

602 if self.config.doAstrometry: 

603 astromRes = self.astrometry.run( 

604 exposure=exposure, 

605 sourceCat=sourceCat, 

606 ) 

607 astromMatches = astromRes.matches 

608 matchMeta = astromRes.matchMeta 

609 if exposure.getWcs() is None: 

610 if self.config.requireAstrometry: 

611 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

612 "is True.") 

613 else: 

614 self.log.warning("Unable to perform astrometric calibration for %r but " 

615 "requireAstrometry is False: attempting to proceed...", 

616 idGenerator) 

617 

618 # compute photometric calibration 

619 if self.config.doPhotoCal: 

620 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

621 if self.config.requirePhotoCal: 

622 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

623 "photoCal, but requirePhotoCal is True.") 

624 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

625 "is False, so skipping photometric calibration and setting photoCalib " 

626 "to None. Attempting to proceed...", idGenerator) 

627 exposure.setPhotoCalib(None) 

628 self.setMetadata(exposure=exposure, photoRes=None) 

629 else: 

630 try: 

631 photoRes = self.photoCal.run( 

632 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

633 ) 

634 exposure.setPhotoCalib(photoRes.photoCalib) 

635 # TODO: reword this to phrase it in terms of the 

636 # calibration factor? 

637 self.log.info("Photometric zero-point: %f", 

638 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

639 self.setMetadata(exposure=exposure, photoRes=photoRes) 

640 except Exception as e: 

641 if self.config.requirePhotoCal: 

642 raise 

643 self.log.warning("Unable to perform photometric calibration " 

644 "(%s): attempting to proceed", e) 

645 self.setMetadata(exposure=exposure, photoRes=None) 

646 

647 self.postCalibrationMeasurement.run( 

648 measCat=sourceCat, 

649 exposure=exposure, 

650 exposureId=idGenerator.catalog_id, 

651 ) 

652 

653 if self.config.doComputeSummaryStats: 

654 summary = self.computeSummaryStats.run(exposure=exposure, 

655 sources=sourceCat, 

656 background=background) 

657 exposure.getInfo().setSummaryStats(summary) 

658 

659 frame = getDebugFrame(self._display, "calibrate") 

660 if frame: 

661 displayAstrometry( 

662 sourceCat=sourceCat, 

663 exposure=exposure, 

664 matches=astromMatches, 

665 frame=frame, 

666 pause=False, 

667 ) 

668 

669 return pipeBase.Struct( 

670 sourceCat=sourceCat, 

671 astromMatches=astromMatches, 

672 matchMeta=matchMeta, 

673 outputExposure=exposure, 

674 outputCat=sourceCat, 

675 outputBackground=background, 

676 ) 

677 

678 def setMetadata(self, exposure, photoRes=None): 

679 """Set task and exposure metadata. 

680 

681 Logs a warning continues if needed data is missing. 

682 

683 Parameters 

684 ---------- 

685 exposure : `lsst.afw.image.ExposureF` 

686 Exposure to set metadata on. 

687 photoRes : `lsst.pipe.base.Struct`, optional 

688 Result of running photoCal task. 

689 """ 

690 if photoRes is None: 

691 return 

692 

693 metadata = exposure.getMetadata() 

694 

695 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

696 try: 

697 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

698 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

699 except Exception: 

700 self.log.warning("Could not set normalized MAGZERO in header: no " 

701 "exposure time") 

702 magZero = math.nan 

703 

704 try: 

705 metadata.set('MAGZERO', magZero) 

706 metadata.set('MAGZERO_RMS', photoRes.sigma) 

707 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

708 metadata.set('COLORTERM1', 0.0) 

709 metadata.set('COLORTERM2', 0.0) 

710 metadata.set('COLORTERM3', 0.0) 

711 except Exception as e: 

712 self.log.warning("Could not set exposure metadata: %s", e) 

713 

714 def copyIcSourceFields(self, icSourceCat, sourceCat): 

715 """Match sources in an icSourceCat and a sourceCat and copy fields. 

716 

717 The fields copied are those specified by 

718 ``config.icSourceFieldsToCopy``. 

719 

720 Parameters 

721 ---------- 

722 icSourceCat : `lsst.afw.table.SourceCatalog` 

723 Catalog from which to copy fields. 

724 sourceCat : `lsst.afw.table.SourceCatalog` 

725 Catalog to which to copy fields. 

726 

727 Raises 

728 ------ 

729 RuntimeError 

730 Raised if any of the following occur: 

731 - icSourceSchema and icSourceKeys are not specified. 

732 - icSourceCat and sourceCat are not specified. 

733 - icSourceFieldsToCopy is empty. 

734 """ 

735 if self.schemaMapper is None: 

736 raise RuntimeError("To copy icSource fields you must specify " 

737 "icSourceSchema and icSourceKeys when " 

738 "constructing this task") 

739 if icSourceCat is None or sourceCat is None: 

740 raise RuntimeError("icSourceCat and sourceCat must both be " 

741 "specified") 

742 if len(self.config.icSourceFieldsToCopy) == 0: 

743 self.log.warning("copyIcSourceFields doing nothing because " 

744 "icSourceFieldsToCopy is empty") 

745 return 

746 

747 mc = afwTable.MatchControl() 

748 mc.findOnlyClosest = False # return all matched objects 

749 matches = afwTable.matchXy(icSourceCat, sourceCat, 

750 self.config.matchRadiusPix, mc) 

751 if self.config.doDeblend: 

752 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

753 # if deblended, keep children 

754 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

755 

756 # Because we had to allow multiple matches to handle parents, we now 

757 # need to prune to the best matches 

758 # closest matches as a dict of icSourceCat source ID: 

759 # (icSourceCat source, sourceCat source, distance in pixels) 

760 bestMatches = {} 

761 for m0, m1, d in matches: 

762 id0 = m0.getId() 

763 match = bestMatches.get(id0) 

764 if match is None or d <= match[2]: 

765 bestMatches[id0] = (m0, m1, d) 

766 matches = list(bestMatches.values()) 

767 

768 # Check that no sourceCat sources are listed twice (we already know 

769 # that each match has a unique icSourceCat source ID, due to using 

770 # that ID as the key in bestMatches) 

771 numMatches = len(matches) 

772 numUniqueSources = len(set(m[1].getId() for m in matches)) 

773 if numUniqueSources != numMatches: 

774 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

775 "sources", numMatches, numUniqueSources) 

776 

777 self.log.info("Copying flags from icSourceCat to sourceCat for " 

778 "%d sources", numMatches) 

779 

780 # For each match: set the calibSourceKey flag and copy the desired 

781 # fields 

782 for icSrc, src, d in matches: 

783 src.setFlag(self.calibSourceKey, True) 

784 # src.assign copies the footprint from icSrc, which we don't want 

785 # (DM-407) 

786 # so set icSrc's footprint to src's footprint before src.assign, 

787 # then restore it 

788 icSrcFootprint = icSrc.getFootprint() 

789 try: 

790 icSrc.setFootprint(src.getFootprint()) 

791 src.assign(icSrc, self.schemaMapper) 

792 finally: 

793 icSrc.setFootprint(icSrcFootprint)