Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

297 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 03:09 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import numpy as np 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches, AstrometryError 

33from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

34import lsst.daf.base as dafBase 

35from lsst.afw.math import BackgroundList 

36from lsst.afw.table import SourceTable 

37from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader, SetPrimaryFlagsTask 

38from lsst.meas.base import (SingleFrameMeasurementTask, 

39 ApplyApCorrTask, 

40 CatalogCalculationTask, 

41 IdGenerator, 

42 DetectorVisitIdGeneratorConfig) 

43from lsst.meas.deblender import SourceDeblendTask 

44from lsst.utils.timer import timeMethod 

45from .photoCal import PhotoCalTask 

46from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

47 

48 

49class _EmptyTargetTask(pipeBase.PipelineTask): 

50 """ 

51 This is a placeholder target for CreateSummaryMetrics and must be retargeted at runtime. 

52 CreateSummaryMetrics should target an analysis tool task, but that would, at the time 

53 of writing, result in a circular import. 

54 

55 As a result, this class should not be used for anything else. 

56 """ 

57 ConfigClass = pipeBase.PipelineTaskConfig 

58 

59 def __init__(self, **kwargs) -> None: 

60 raise NotImplementedError( 

61 "doCreateSummaryMetrics is set to True, in which case " 

62 "createSummaryMetrics must be retargeted." 

63 ) 

64 

65 

66class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

67 defaultTemplates={}): 

68 

69 icSourceSchema = cT.InitInput( 

70 doc="Schema produced by characterize image task, used to initialize this task", 

71 name="icSrc_schema", 

72 storageClass="SourceCatalog", 

73 ) 

74 

75 outputSchema = cT.InitOutput( 

76 doc="Schema after CalibrateTask has been initialized", 

77 name="src_schema", 

78 storageClass="SourceCatalog", 

79 ) 

80 

81 exposure = cT.Input( 

82 doc="Input image to calibrate", 

83 name="icExp", 

84 storageClass="ExposureF", 

85 dimensions=("instrument", "visit", "detector"), 

86 ) 

87 

88 background = cT.Input( 

89 doc="Backgrounds determined by characterize task", 

90 name="icExpBackground", 

91 storageClass="Background", 

92 dimensions=("instrument", "visit", "detector"), 

93 ) 

94 

95 icSourceCat = cT.Input( 

96 doc="Source catalog created by characterize task", 

97 name="icSrc", 

98 storageClass="SourceCatalog", 

99 dimensions=("instrument", "visit", "detector"), 

100 ) 

101 

102 astromRefCat = cT.PrerequisiteInput( 

103 doc="Reference catalog to use for astrometry", 

104 name="gaia_dr3_20230707", 

105 storageClass="SimpleCatalog", 

106 dimensions=("skypix",), 

107 deferLoad=True, 

108 multiple=True, 

109 ) 

110 

111 photoRefCat = cT.PrerequisiteInput( 

112 doc="Reference catalog to use for photometric calibration", 

113 name="ps1_pv3_3pi_20170110", 

114 storageClass="SimpleCatalog", 

115 dimensions=("skypix",), 

116 deferLoad=True, 

117 multiple=True 

118 ) 

119 

120 outputExposure = cT.Output( 

121 doc="Exposure after running calibration task", 

122 name="calexp", 

123 storageClass="ExposureF", 

124 dimensions=("instrument", "visit", "detector"), 

125 ) 

126 

127 outputCat = cT.Output( 

128 doc="Source catalog produced in calibrate task", 

129 name="src", 

130 storageClass="SourceCatalog", 

131 dimensions=("instrument", "visit", "detector"), 

132 ) 

133 

134 outputBackground = cT.Output( 

135 doc="Background models estimated in calibration task", 

136 name="calexpBackground", 

137 storageClass="Background", 

138 dimensions=("instrument", "visit", "detector"), 

139 ) 

140 

141 outputSummaryMetrics = cT.Output( 

142 doc="Summary metrics created by the calibration task", 

143 name="calexpSummary_metrics", 

144 storageClass="MetricMeasurementBundle", 

145 dimensions=("instrument", "visit", "detector"), 

146 ) 

147 

148 matches = cT.Output( 

149 doc="Source/refObj matches from the astrometry solver", 

150 name="srcMatch", 

151 storageClass="Catalog", 

152 dimensions=("instrument", "visit", "detector"), 

153 ) 

154 

155 matchesDenormalized = cT.Output( 

156 doc="Denormalized matches from astrometry solver", 

157 name="srcMatchFull", 

158 storageClass="Catalog", 

159 dimensions=("instrument", "visit", "detector"), 

160 ) 

161 

162 def __init__(self, *, config=None): 

163 super().__init__(config=config) 

164 

165 if config.doAstrometry is False: 

166 self.prerequisiteInputs.remove("astromRefCat") 

167 if config.doPhotoCal is False: 

168 self.prerequisiteInputs.remove("photoRefCat") 

169 

170 if config.doWriteMatches is False or config.doAstrometry is False: 

171 self.outputs.remove("matches") 

172 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

173 self.outputs.remove("matchesDenormalized") 

174 

175 if config.doCreateSummaryMetrics is False: 

176 self.outputs.remove("outputSummaryMetrics") 

177 

178 

179class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

180 """Config for CalibrateTask.""" 

181 

182 doWrite = pexConfig.Field( 

183 dtype=bool, 

184 default=True, 

185 doc="Save calibration results?", 

186 ) 

187 doWriteHeavyFootprintsInSources = pexConfig.Field( 

188 dtype=bool, 

189 default=True, 

190 doc="Include HeavyFootprint data in source table? If false then heavy " 

191 "footprints are saved as normal footprints, which saves some space" 

192 ) 

193 doWriteMatches = pexConfig.Field( 

194 dtype=bool, 

195 default=True, 

196 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

197 ) 

198 doWriteMatchesDenormalized = pexConfig.Field( 

199 dtype=bool, 

200 default=True, 

201 doc=("Write reference matches in denormalized format? " 

202 "This format uses more disk space, but is more convenient to " 

203 "read for debugging. Ignored if doWriteMatches=False or doWrite=False."), 

204 ) 

205 doAstrometry = pexConfig.Field( 

206 dtype=bool, 

207 default=True, 

208 doc="Perform astrometric calibration?", 

209 ) 

210 astromRefObjLoader = pexConfig.ConfigField( 

211 dtype=LoadReferenceObjectsConfig, 

212 doc="reference object loader for astrometric calibration", 

213 ) 

214 photoRefObjLoader = pexConfig.ConfigField( 

215 dtype=LoadReferenceObjectsConfig, 

216 doc="reference object loader for photometric calibration", 

217 ) 

218 astrometry = pexConfig.ConfigurableField( 

219 target=AstrometryTask, 

220 doc="Perform astrometric calibration to refine the WCS", 

221 ) 

222 requireAstrometry = pexConfig.Field( 

223 dtype=bool, 

224 default=True, 

225 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

226 "false."), 

227 ) 

228 doPhotoCal = pexConfig.Field( 

229 dtype=bool, 

230 default=True, 

231 doc="Perform phometric calibration?", 

232 ) 

233 requirePhotoCal = pexConfig.Field( 

234 dtype=bool, 

235 default=True, 

236 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

237 "false."), 

238 ) 

239 photoCal = pexConfig.ConfigurableField( 

240 target=PhotoCalTask, 

241 doc="Perform photometric calibration", 

242 ) 

243 icSourceFieldsToCopy = pexConfig.ListField( 

244 dtype=str, 

245 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

246 doc=("Fields to copy from the icSource catalog to the output catalog " 

247 "for matching sources Any missing fields will trigger a " 

248 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

249 ) 

250 matchRadiusPix = pexConfig.Field( 

251 dtype=float, 

252 default=3, 

253 doc=("Match radius for matching icSourceCat objects to sourceCat " 

254 "objects (pixels)"), 

255 ) 

256 checkUnitsParseStrict = pexConfig.Field( 

257 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

258 "'warn' or 'silent'"), 

259 dtype=str, 

260 default="raise", 

261 ) 

262 detection = pexConfig.ConfigurableField( 

263 target=SourceDetectionTask, 

264 doc="Detect sources" 

265 ) 

266 doDeblend = pexConfig.Field( 

267 dtype=bool, 

268 default=True, 

269 doc="Run deblender input exposure" 

270 ) 

271 deblend = pexConfig.ConfigurableField( 

272 target=SourceDeblendTask, 

273 doc="Split blended sources into their components" 

274 ) 

275 doSkySources = pexConfig.Field( 

276 dtype=bool, 

277 default=True, 

278 doc="Generate sky sources?", 

279 ) 

280 skySources = pexConfig.ConfigurableField( 

281 target=SkyObjectsTask, 

282 doc="Generate sky sources", 

283 ) 

284 measurement = pexConfig.ConfigurableField( 

285 target=SingleFrameMeasurementTask, 

286 doc="Measure sources" 

287 ) 

288 postCalibrationMeasurement = pexConfig.ConfigurableField( 

289 target=SingleFrameMeasurementTask, 

290 doc="Second round of measurement for plugins that need to be run after photocal" 

291 ) 

292 setPrimaryFlags = pexConfig.ConfigurableField( 

293 target=SetPrimaryFlagsTask, 

294 doc=("Set flags for primary source classification in single frame " 

295 "processing. True if sources are not sky sources and not a parent.") 

296 ) 

297 doApCorr = pexConfig.Field( 

298 dtype=bool, 

299 default=True, 

300 doc="Run subtask to apply aperture correction" 

301 ) 

302 applyApCorr = pexConfig.ConfigurableField( 

303 target=ApplyApCorrTask, 

304 doc="Subtask to apply aperture corrections" 

305 ) 

306 # If doApCorr is False, and the exposure does not have apcorrections 

307 # already applied, the active plugins in catalogCalculation almost 

308 # certainly should not contain the characterization plugin 

309 catalogCalculation = pexConfig.ConfigurableField( 

310 target=CatalogCalculationTask, 

311 doc="Subtask to run catalogCalculation plugins on catalog" 

312 ) 

313 doComputeSummaryStats = pexConfig.Field( 

314 dtype=bool, 

315 default=True, 

316 doc="Run subtask to measure exposure summary statistics?" 

317 ) 

318 computeSummaryStats = pexConfig.ConfigurableField( 

319 target=ComputeExposureSummaryStatsTask, 

320 doc="Subtask to run computeSummaryStats on exposure" 

321 ) 

322 doCreateSummaryMetrics = pexConfig.Field( 

323 dtype=bool, 

324 default=False, 

325 doc="Run the subtask to create summary metrics, and then write those metrics." 

326 ) 

327 createSummaryMetrics = pexConfig.ConfigurableField( 

328 target=_EmptyTargetTask, 

329 doc="Subtask to create metrics from the summary stats. This must be retargeted, likely to an" 

330 "analysis_tools task such as CalexpSummaryMetrics." 

331 ) 

332 doWriteExposure = pexConfig.Field( 

333 dtype=bool, 

334 default=True, 

335 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

336 "normal calexp but as a fakes_calexp." 

337 ) 

338 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

339 

340 def setDefaults(self): 

341 super().setDefaults() 

342 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

343 self.postCalibrationMeasurement.doReplaceWithNoise = False 

344 for key in self.postCalibrationMeasurement.slots: 

345 setattr(self.postCalibrationMeasurement.slots, key, None) 

346 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

347 # The photoRefCat connection is the name to use for the colorterms. 

348 self.photoCal.photoCatName = self.connections.photoRefCat 

349 

350 # Keep track of which footprints contain streaks 

351 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

352 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

353 

354 

355class CalibrateTask(pipeBase.PipelineTask): 

356 """Calibrate an exposure: measure sources and perform astrometric and 

357 photometric calibration. 

358 

359 Given an exposure with a good PSF model and aperture correction map(e.g. as 

360 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

361 perform the following operations: 

362 - Run detection and measurement 

363 - Run astrometry subtask to fit an improved WCS 

364 - Run photoCal subtask to fit the exposure's photometric zero-point 

365 

366 Parameters 

367 ---------- 

368 butler : `None` 

369 Compatibility parameter. Should always be `None`. 

370 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

371 Unused in gen3: must be `None`. 

372 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

373 Unused in gen3: must be `None`. 

374 icSourceSchema : `lsst.afw.table.Schema`, optional 

375 Schema for the icSource catalog. 

376 initInputs : `dict`, optional 

377 Dictionary that can contain a key ``icSourceSchema`` containing the 

378 input schema. If present will override the value of ``icSourceSchema``. 

379 

380 Raises 

381 ------ 

382 RuntimeError 

383 Raised if any of the following occur: 

384 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

385 - PipelineTask form of this task is initialized with reference object 

386 loaders. 

387 

388 Notes 

389 ----- 

390 Quantities set in exposure Metadata: 

391 

392 MAGZERO_RMS 

393 MAGZERO's RMS == sigma reported by photoCal task 

394 MAGZERO_NOBJ 

395 Number of stars used == ngood reported by photoCal task 

396 COLORTERM1 

397 ?? (always 0.0) 

398 COLORTERM2 

399 ?? (always 0.0) 

400 COLORTERM3 

401 ?? (always 0.0) 

402 

403 Debugging: 

404 CalibrateTask has a debug dictionary containing one key: 

405 

406 calibrate 

407 frame (an int; <= 0 to not display) in which to display the exposure, 

408 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

409 the meaning of the various symbols. 

410 """ 

411 

412 ConfigClass = CalibrateConfig 

413 _DefaultName = "calibrate" 

414 

415 def __init__(self, astromRefObjLoader=None, 

416 photoRefObjLoader=None, icSourceSchema=None, 

417 initInputs=None, **kwargs): 

418 super().__init__(**kwargs) 

419 

420 if initInputs is not None: 

421 icSourceSchema = initInputs['icSourceSchema'].schema 

422 

423 if icSourceSchema is not None: 

424 # use a schema mapper to avoid copying each field separately 

425 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

426 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

427 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

428 

429 # Add fields to copy from an icSource catalog 

430 # and a field to indicate that the source matched a source in that 

431 # catalog. If any fields are missing then raise an exception, but 

432 # first find all missing fields in order to make the error message 

433 # more useful. 

434 self.calibSourceKey = self.schemaMapper.addOutputField( 

435 afwTable.Field["Flag"]("calib_detected", 

436 "Source was detected as an icSource")) 

437 missingFieldNames = [] 

438 for fieldName in self.config.icSourceFieldsToCopy: 

439 try: 

440 schemaItem = icSourceSchema.find(fieldName) 

441 except Exception: 

442 missingFieldNames.append(fieldName) 

443 else: 

444 # field found; if addMapping fails then raise an exception 

445 self.schemaMapper.addMapping(schemaItem.getKey()) 

446 

447 if missingFieldNames: 

448 raise RuntimeError("isSourceCat is missing fields {} " 

449 "specified in icSourceFieldsToCopy" 

450 .format(missingFieldNames)) 

451 

452 # produce a temporary schema to pass to the subtasks; finalize it 

453 # later 

454 self.schema = self.schemaMapper.editOutputSchema() 

455 else: 

456 self.schemaMapper = None 

457 self.schema = afwTable.SourceTable.makeMinimalSchema() 

458 afwTable.CoordKey.addErrorFields(self.schema) 

459 self.makeSubtask('detection', schema=self.schema) 

460 

461 self.algMetadata = dafBase.PropertyList() 

462 

463 if self.config.doDeblend: 

464 self.makeSubtask("deblend", schema=self.schema) 

465 if self.config.doSkySources: 

466 self.makeSubtask("skySources") 

467 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

468 self.makeSubtask('measurement', schema=self.schema, 

469 algMetadata=self.algMetadata) 

470 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

471 algMetadata=self.algMetadata) 

472 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

473 if self.config.doApCorr: 

474 self.makeSubtask('applyApCorr', schema=self.schema) 

475 self.makeSubtask('catalogCalculation', schema=self.schema) 

476 

477 if self.config.doAstrometry: 

478 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

479 schema=self.schema) 

480 if self.config.doPhotoCal: 

481 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

482 schema=self.schema) 

483 if self.config.doComputeSummaryStats: 

484 self.makeSubtask('computeSummaryStats') 

485 if self.config.doCreateSummaryMetrics: 

486 self.makeSubtask('createSummaryMetrics') 

487 

488 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

489 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

490 "reference object loaders.") 

491 

492 if self.schemaMapper is not None: 

493 # finalize the schema 

494 self.schema = self.schemaMapper.getOutputSchema() 

495 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

496 

497 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

498 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

499 self.outputSchema = sourceCatSchema 

500 

501 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

502 inputs = butlerQC.get(inputRefs) 

503 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

504 

505 if self.config.doAstrometry: 

506 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

507 for ref in inputRefs.astromRefCat], 

508 refCats=inputs.pop('astromRefCat'), 

509 name=self.config.connections.astromRefCat, 

510 config=self.config.astromRefObjLoader, log=self.log) 

511 self.astrometry.setRefObjLoader(refObjLoader) 

512 

513 if self.config.doPhotoCal: 

514 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

515 for ref in inputRefs.photoRefCat], 

516 refCats=inputs.pop('photoRefCat'), 

517 name=self.config.connections.photoRefCat, 

518 config=self.config.photoRefObjLoader, 

519 log=self.log) 

520 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

521 

522 outputs = self.run(**inputs) 

523 

524 if self.config.doWriteMatches and self.config.doAstrometry: 

525 if outputs.astromMatches is not None: 

526 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

527 normalizedMatches.table.setMetadata(outputs.matchMeta) 

528 if self.config.doWriteMatchesDenormalized: 

529 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

530 outputs.matchesDenormalized = denormMatches 

531 outputs.matches = normalizedMatches 

532 else: 

533 del outputRefs.matches 

534 if self.config.doWriteMatchesDenormalized: 

535 del outputRefs.matchesDenormalized 

536 butlerQC.put(outputs, outputRefs) 

537 

538 @timeMethod 

539 def run(self, exposure, background=None, 

540 icSourceCat=None, idGenerator=None): 

541 """Calibrate an exposure. 

542 

543 Parameters 

544 ---------- 

545 exposure : `lsst.afw.image.ExposureF` 

546 Exposure to calibrate. 

547 background : `lsst.afw.math.BackgroundList`, optional 

548 Initial model of background already subtracted from exposure. 

549 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

550 SourceCatalog from CharacterizeImageTask from which we can copy 

551 some fields. 

552 idGenerator : `lsst.meas.base.IdGenerator`, optional 

553 Object that generates source IDs and provides RNG seeds. 

554 

555 Returns 

556 ------- 

557 result : `lsst.pipe.base.Struct` 

558 Results as a struct with attributes: 

559 

560 ``exposure`` 

561 Characterized exposure (`lsst.afw.image.ExposureF`). 

562 ``sourceCat`` 

563 Detected sources (`lsst.afw.table.SourceCatalog`). 

564 ``outputBackground`` 

565 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

566 ``astromMatches`` 

567 List of source/ref matches from astrometry solver. 

568 ``matchMeta`` 

569 Metadata from astrometry matches. 

570 ``outputExposure`` 

571 Another reference to ``exposure`` for compatibility. 

572 ``outputCat`` 

573 Another reference to ``sourceCat`` for compatibility. 

574 """ 

575 # detect, deblend and measure sources 

576 if idGenerator is None: 

577 idGenerator = IdGenerator() 

578 

579 if background is None: 

580 background = BackgroundList() 

581 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

582 table.setMetadata(self.algMetadata) 

583 

584 detRes = self.detection.run(table=table, exposure=exposure, 

585 doSmooth=True) 

586 sourceCat = detRes.sources 

587 if detRes.background: 

588 for bg in detRes.background: 

589 background.append(bg) 

590 if self.config.doSkySources: 

591 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

592 if skySourceFootprints: 

593 for foot in skySourceFootprints: 

594 s = sourceCat.addNew() 

595 s.setFootprint(foot) 

596 s.set(self.skySourceKey, True) 

597 if self.config.doDeblend: 

598 self.deblend.run(exposure=exposure, sources=sourceCat) 

599 if not sourceCat.isContiguous(): 

600 sourceCat = sourceCat.copy(deep=True) 

601 self.measurement.run( 

602 measCat=sourceCat, 

603 exposure=exposure, 

604 exposureId=idGenerator.catalog_id, 

605 ) 

606 if self.config.doApCorr: 

607 apCorrMap = exposure.getInfo().getApCorrMap() 

608 if apCorrMap is None: 

609 self.log.warning("Image does not have valid aperture correction map for %r; " 

610 "skipping aperture correction", idGenerator) 

611 else: 

612 self.applyApCorr.run( 

613 catalog=sourceCat, 

614 apCorrMap=apCorrMap, 

615 ) 

616 self.catalogCalculation.run(sourceCat) 

617 

618 self.setPrimaryFlags.run(sourceCat) 

619 

620 if icSourceCat is not None and \ 

621 len(self.config.icSourceFieldsToCopy) > 0: 

622 self.copyIcSourceFields(icSourceCat=icSourceCat, 

623 sourceCat=sourceCat) 

624 

625 # TODO DM-11568: this contiguous check-and-copy could go away if we 

626 # reserve enough space during SourceDetection and/or SourceDeblend. 

627 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

628 # contiguity now, so views are preserved from here on. 

629 if not sourceCat.isContiguous(): 

630 sourceCat = sourceCat.copy(deep=True) 

631 

632 # perform astrometry calibration: 

633 # fit an improved WCS and update the exposure's WCS in place 

634 astromMatches = None 

635 matchMeta = None 

636 if self.config.doAstrometry: 

637 try: 

638 astromRes = self.astrometry.run(exposure=exposure, sourceCat=sourceCat) 

639 astromMatches = astromRes.matches 

640 matchMeta = astromRes.matchMeta 

641 except AstrometryError as e: 

642 # Maintain old behavior of not stopping for astrometry errors. 

643 self.log.warning(e) 

644 if exposure.getWcs() is None: 

645 if self.config.requireAstrometry: 

646 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

647 "is True.") 

648 else: 

649 self.log.warning("Unable to perform astrometric calibration for %r but " 

650 "requireAstrometry is False: attempting to proceed...", 

651 idGenerator) 

652 

653 # compute photometric calibration 

654 if self.config.doPhotoCal: 

655 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

656 if self.config.requirePhotoCal: 

657 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

658 "photoCal, but requirePhotoCal is True.") 

659 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

660 "is False, so skipping photometric calibration and setting photoCalib " 

661 "to None. Attempting to proceed...", idGenerator) 

662 exposure.setPhotoCalib(None) 

663 self.setMetadata(exposure=exposure, photoRes=None) 

664 else: 

665 try: 

666 photoRes = self.photoCal.run( 

667 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

668 ) 

669 exposure.setPhotoCalib(photoRes.photoCalib) 

670 # TODO: reword this to phrase it in terms of the 

671 # calibration factor? 

672 self.log.info("Photometric zero-point: %f", 

673 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

674 self.setMetadata(exposure=exposure, photoRes=photoRes) 

675 except Exception as e: 

676 if self.config.requirePhotoCal: 

677 raise 

678 self.log.warning("Unable to perform photometric calibration " 

679 "(%s): attempting to proceed", e) 

680 self.setMetadata(exposure=exposure, photoRes=None) 

681 

682 self.postCalibrationMeasurement.run( 

683 measCat=sourceCat, 

684 exposure=exposure, 

685 exposureId=idGenerator.catalog_id, 

686 ) 

687 

688 summaryMetrics = None 

689 if self.config.doComputeSummaryStats: 

690 summary = self.computeSummaryStats.run(exposure=exposure, 

691 sources=sourceCat, 

692 background=background) 

693 exposure.getInfo().setSummaryStats(summary) 

694 if self.config.doCreateSummaryMetrics: 

695 summaryMetrics = self.createSummaryMetrics.run(data=summary.__dict__).metrics 

696 

697 frame = getDebugFrame(self._display, "calibrate") 

698 if frame: 

699 displayAstrometry( 

700 sourceCat=sourceCat, 

701 exposure=exposure, 

702 matches=astromMatches, 

703 frame=frame, 

704 pause=False, 

705 ) 

706 

707 return pipeBase.Struct( 

708 sourceCat=sourceCat, 

709 astromMatches=astromMatches, 

710 matchMeta=matchMeta, 

711 outputExposure=exposure, 

712 outputCat=sourceCat, 

713 outputBackground=background, 

714 outputSummaryMetrics=summaryMetrics 

715 ) 

716 

717 def setMetadata(self, exposure, photoRes=None): 

718 """Set task and exposure metadata. 

719 

720 Logs a warning continues if needed data is missing. 

721 

722 Parameters 

723 ---------- 

724 exposure : `lsst.afw.image.ExposureF` 

725 Exposure to set metadata on. 

726 photoRes : `lsst.pipe.base.Struct`, optional 

727 Result of running photoCal task. 

728 """ 

729 if photoRes is None: 

730 return 

731 

732 metadata = exposure.getMetadata() 

733 

734 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

735 try: 

736 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

737 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

738 except Exception: 

739 self.log.warning("Could not set normalized MAGZERO in header: no " 

740 "exposure time") 

741 magZero = math.nan 

742 

743 try: 

744 metadata.set('MAGZERO', magZero) 

745 metadata.set('MAGZERO_RMS', photoRes.sigma) 

746 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

747 metadata.set('COLORTERM1', 0.0) 

748 metadata.set('COLORTERM2', 0.0) 

749 metadata.set('COLORTERM3', 0.0) 

750 except Exception as e: 

751 self.log.warning("Could not set exposure metadata: %s", e) 

752 

753 def copyIcSourceFields(self, icSourceCat, sourceCat): 

754 """Match sources in an icSourceCat and a sourceCat and copy fields. 

755 

756 The fields copied are those specified by 

757 ``config.icSourceFieldsToCopy``. 

758 

759 Parameters 

760 ---------- 

761 icSourceCat : `lsst.afw.table.SourceCatalog` 

762 Catalog from which to copy fields. 

763 sourceCat : `lsst.afw.table.SourceCatalog` 

764 Catalog to which to copy fields. 

765 

766 Raises 

767 ------ 

768 RuntimeError 

769 Raised if any of the following occur: 

770 - icSourceSchema and icSourceKeys are not specified. 

771 - icSourceCat and sourceCat are not specified. 

772 - icSourceFieldsToCopy is empty. 

773 """ 

774 if self.schemaMapper is None: 

775 raise RuntimeError("To copy icSource fields you must specify " 

776 "icSourceSchema and icSourceKeys when " 

777 "constructing this task") 

778 if icSourceCat is None or sourceCat is None: 

779 raise RuntimeError("icSourceCat and sourceCat must both be " 

780 "specified") 

781 if len(self.config.icSourceFieldsToCopy) == 0: 

782 self.log.warning("copyIcSourceFields doing nothing because " 

783 "icSourceFieldsToCopy is empty") 

784 return 

785 

786 mc = afwTable.MatchControl() 

787 mc.findOnlyClosest = False # return all matched objects 

788 matches = afwTable.matchXy(icSourceCat, sourceCat, 

789 self.config.matchRadiusPix, mc) 

790 if self.config.doDeblend: 

791 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

792 # if deblended, keep children 

793 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

794 

795 # Because we had to allow multiple matches to handle parents, we now 

796 # need to prune to the best matches 

797 # closest matches as a dict of icSourceCat source ID: 

798 # (icSourceCat source, sourceCat source, distance in pixels) 

799 bestMatches = {} 

800 for m0, m1, d in matches: 

801 id0 = m0.getId() 

802 match = bestMatches.get(id0) 

803 if match is None or d <= match[2]: 

804 bestMatches[id0] = (m0, m1, d) 

805 matches = list(bestMatches.values()) 

806 

807 # Check that no sourceCat sources are listed twice (we already know 

808 # that each match has a unique icSourceCat source ID, due to using 

809 # that ID as the key in bestMatches) 

810 numMatches = len(matches) 

811 numUniqueSources = len(set(m[1].getId() for m in matches)) 

812 if numUniqueSources != numMatches: 

813 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

814 "sources", numMatches, numUniqueSources) 

815 

816 self.log.info("Copying flags from icSourceCat to sourceCat for " 

817 "%d sources", numMatches) 

818 

819 # For each match: set the calibSourceKey flag and copy the desired 

820 # fields 

821 for icSrc, src, d in matches: 

822 src.setFlag(self.calibSourceKey, True) 

823 # src.assign copies the footprint from icSrc, which we don't want 

824 # (DM-407) 

825 # so set icSrc's footprint to src's footprint before src.assign, 

826 # then restore it 

827 icSrcFootprint = icSrc.getFootprint() 

828 try: 

829 icSrc.setFootprint(src.getFootprint()) 

830 src.assign(icSrc, self.schemaMapper) 

831 finally: 

832 icSrc.setFootprint(icSrcFootprint)