Coverage for python/lsst/pipe/tasks/calibrate.py: 19%

280 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 10:21 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import numpy as np 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

33from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

34import lsst.daf.base as dafBase 

35from lsst.afw.math import BackgroundList 

36from lsst.afw.table import SourceTable 

37from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

38from lsst.meas.base import (SingleFrameMeasurementTask, 

39 ApplyApCorrTask, 

40 CatalogCalculationTask, 

41 IdGenerator, 

42 DetectorVisitIdGeneratorConfig) 

43from lsst.meas.deblender import SourceDeblendTask 

44from lsst.utils.timer import timeMethod 

45from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

46from .photoCal import PhotoCalTask 

47from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="gaia_dr2_20200414", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="ps1_pv3_3pi_20170110", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask.""" 

155 

156 doWrite = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Save calibration results?", 

160 ) 

161 doWriteHeavyFootprintsInSources = pexConfig.Field( 

162 dtype=bool, 

163 default=True, 

164 doc="Include HeavyFootprint data in source table? If false then heavy " 

165 "footprints are saved as normal footprints, which saves some space" 

166 ) 

167 doWriteMatches = pexConfig.Field( 

168 dtype=bool, 

169 default=True, 

170 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

171 ) 

172 doWriteMatchesDenormalized = pexConfig.Field( 

173 dtype=bool, 

174 default=False, 

175 doc=("Write reference matches in denormalized format? " 

176 "This format uses more disk space, but is more convenient to " 

177 "read. Ignored if doWriteMatches=False or doWrite=False."), 

178 ) 

179 doAstrometry = pexConfig.Field( 

180 dtype=bool, 

181 default=True, 

182 doc="Perform astrometric calibration?", 

183 ) 

184 astromRefObjLoader = pexConfig.ConfigField( 

185 dtype=LoadReferenceObjectsConfig, 

186 doc="reference object loader for astrometric calibration", 

187 ) 

188 photoRefObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 doc="reference object loader for photometric calibration", 

191 ) 

192 astrometry = pexConfig.ConfigurableField( 

193 target=AstrometryTask, 

194 doc="Perform astrometric calibration to refine the WCS", 

195 ) 

196 requireAstrometry = pexConfig.Field( 

197 dtype=bool, 

198 default=True, 

199 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

200 "false."), 

201 ) 

202 doPhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc="Perform phometric calibration?", 

206 ) 

207 requirePhotoCal = pexConfig.Field( 

208 dtype=bool, 

209 default=True, 

210 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

211 "false."), 

212 ) 

213 photoCal = pexConfig.ConfigurableField( 

214 target=PhotoCalTask, 

215 doc="Perform photometric calibration", 

216 ) 

217 icSourceFieldsToCopy = pexConfig.ListField( 

218 dtype=str, 

219 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

220 doc=("Fields to copy from the icSource catalog to the output catalog " 

221 "for matching sources Any missing fields will trigger a " 

222 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

223 ) 

224 matchRadiusPix = pexConfig.Field( 

225 dtype=float, 

226 default=3, 

227 doc=("Match radius for matching icSourceCat objects to sourceCat " 

228 "objects (pixels)"), 

229 ) 

230 checkUnitsParseStrict = pexConfig.Field( 

231 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

232 "'warn' or 'silent'"), 

233 dtype=str, 

234 default="raise", 

235 ) 

236 detection = pexConfig.ConfigurableField( 

237 target=SourceDetectionTask, 

238 doc="Detect sources" 

239 ) 

240 doDeblend = pexConfig.Field( 

241 dtype=bool, 

242 default=True, 

243 doc="Run deblender input exposure" 

244 ) 

245 deblend = pexConfig.ConfigurableField( 

246 target=SourceDeblendTask, 

247 doc="Split blended sources into their components" 

248 ) 

249 doSkySources = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Generate sky sources?", 

253 ) 

254 skySources = pexConfig.ConfigurableField( 

255 target=SkyObjectsTask, 

256 doc="Generate sky sources", 

257 ) 

258 measurement = pexConfig.ConfigurableField( 

259 target=SingleFrameMeasurementTask, 

260 doc="Measure sources" 

261 ) 

262 postCalibrationMeasurement = pexConfig.ConfigurableField( 

263 target=SingleFrameMeasurementTask, 

264 doc="Second round of measurement for plugins that need to be run after photocal" 

265 ) 

266 setPrimaryFlags = pexConfig.ConfigurableField( 

267 target=SetPrimaryFlagsTask, 

268 doc=("Set flags for primary source classification in single frame " 

269 "processing. True if sources are not sky sources and not a parent.") 

270 ) 

271 doApCorr = pexConfig.Field( 

272 dtype=bool, 

273 default=True, 

274 doc="Run subtask to apply aperture correction" 

275 ) 

276 applyApCorr = pexConfig.ConfigurableField( 

277 target=ApplyApCorrTask, 

278 doc="Subtask to apply aperture corrections" 

279 ) 

280 # If doApCorr is False, and the exposure does not have apcorrections 

281 # already applied, the active plugins in catalogCalculation almost 

282 # certainly should not contain the characterization plugin 

283 catalogCalculation = pexConfig.ConfigurableField( 

284 target=CatalogCalculationTask, 

285 doc="Subtask to run catalogCalculation plugins on catalog" 

286 ) 

287 doComputeSummaryStats = pexConfig.Field( 

288 dtype=bool, 

289 default=True, 

290 doc="Run subtask to measure exposure summary statistics?" 

291 ) 

292 computeSummaryStats = pexConfig.ConfigurableField( 

293 target=ComputeExposureSummaryStatsTask, 

294 doc="Subtask to run computeSummaryStats on exposure" 

295 ) 

296 doWriteExposure = pexConfig.Field( 

297 dtype=bool, 

298 default=True, 

299 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

300 "normal calexp but as a fakes_calexp." 

301 ) 

302 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

303 

304 def setDefaults(self): 

305 super().setDefaults() 

306 self.detection.doTempLocalBackground = False 

307 self.deblend.maxFootprintSize = 2000 

308 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

309 self.postCalibrationMeasurement.doReplaceWithNoise = False 

310 for key in self.postCalibrationMeasurement.slots: 

311 setattr(self.postCalibrationMeasurement.slots, key, None) 

312 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

313 # The photoRefCat connection is the name to use for the colorterms. 

314 self.photoCal.photoCatName = self.connections.photoRefCat 

315 

316 

317class CalibrateTask(pipeBase.PipelineTask): 

318 """Calibrate an exposure: measure sources and perform astrometric and 

319 photometric calibration. 

320 

321 Given an exposure with a good PSF model and aperture correction map(e.g. as 

322 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

323 perform the following operations: 

324 - Run detection and measurement 

325 - Run astrometry subtask to fit an improved WCS 

326 - Run photoCal subtask to fit the exposure's photometric zero-point 

327 

328 Parameters 

329 ---------- 

330 butler : `None` 

331 Compatibility parameter. Should always be `None`. 

332 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

333 Unused in gen3: must be `None`. 

334 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

335 Unused in gen3: must be `None`. 

336 icSourceSchema : `lsst.afw.table.Schema`, optional 

337 Schema for the icSource catalog. 

338 initInputs : `dict`, optional 

339 Dictionary that can contain a key ``icSourceSchema`` containing the 

340 input schema. If present will override the value of ``icSourceSchema``. 

341 

342 Raises 

343 ------ 

344 RuntimeError 

345 Raised if any of the following occur: 

346 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

347 - PipelineTask form of this task is initialized with reference object 

348 loaders. 

349 

350 Notes 

351 ----- 

352 Quantities set in exposure Metadata: 

353 

354 MAGZERO_RMS 

355 MAGZERO's RMS == sigma reported by photoCal task 

356 MAGZERO_NOBJ 

357 Number of stars used == ngood reported by photoCal task 

358 COLORTERM1 

359 ?? (always 0.0) 

360 COLORTERM2 

361 ?? (always 0.0) 

362 COLORTERM3 

363 ?? (always 0.0) 

364 

365 Debugging: 

366 CalibrateTask has a debug dictionary containing one key: 

367 

368 calibrate 

369 frame (an int; <= 0 to not display) in which to display the exposure, 

370 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

371 the meaning of the various symbols. 

372 """ 

373 

374 ConfigClass = CalibrateConfig 

375 _DefaultName = "calibrate" 

376 

377 def __init__(self, astromRefObjLoader=None, 

378 photoRefObjLoader=None, icSourceSchema=None, 

379 initInputs=None, **kwargs): 

380 super().__init__(**kwargs) 

381 

382 if initInputs is not None: 

383 icSourceSchema = initInputs['icSourceSchema'].schema 

384 

385 if icSourceSchema is not None: 

386 # use a schema mapper to avoid copying each field separately 

387 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

388 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

389 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

390 

391 # Add fields to copy from an icSource catalog 

392 # and a field to indicate that the source matched a source in that 

393 # catalog. If any fields are missing then raise an exception, but 

394 # first find all missing fields in order to make the error message 

395 # more useful. 

396 self.calibSourceKey = self.schemaMapper.addOutputField( 

397 afwTable.Field["Flag"]("calib_detected", 

398 "Source was detected as an icSource")) 

399 missingFieldNames = [] 

400 for fieldName in self.config.icSourceFieldsToCopy: 

401 try: 

402 schemaItem = icSourceSchema.find(fieldName) 

403 except Exception: 

404 missingFieldNames.append(fieldName) 

405 else: 

406 # field found; if addMapping fails then raise an exception 

407 self.schemaMapper.addMapping(schemaItem.getKey()) 

408 

409 if missingFieldNames: 

410 raise RuntimeError("isSourceCat is missing fields {} " 

411 "specified in icSourceFieldsToCopy" 

412 .format(missingFieldNames)) 

413 

414 # produce a temporary schema to pass to the subtasks; finalize it 

415 # later 

416 self.schema = self.schemaMapper.editOutputSchema() 

417 else: 

418 self.schemaMapper = None 

419 self.schema = afwTable.SourceTable.makeMinimalSchema() 

420 self.makeSubtask('detection', schema=self.schema) 

421 

422 self.algMetadata = dafBase.PropertyList() 

423 

424 if self.config.doDeblend: 

425 self.makeSubtask("deblend", schema=self.schema) 

426 if self.config.doSkySources: 

427 self.makeSubtask("skySources") 

428 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

429 self.makeSubtask('measurement', schema=self.schema, 

430 algMetadata=self.algMetadata) 

431 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

432 algMetadata=self.algMetadata) 

433 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

434 if self.config.doApCorr: 

435 self.makeSubtask('applyApCorr', schema=self.schema) 

436 self.makeSubtask('catalogCalculation', schema=self.schema) 

437 

438 if self.config.doAstrometry: 

439 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

440 schema=self.schema) 

441 if self.config.doPhotoCal: 

442 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

443 schema=self.schema) 

444 if self.config.doComputeSummaryStats: 

445 self.makeSubtask('computeSummaryStats') 

446 

447 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

448 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

449 "reference object loaders.") 

450 

451 if self.schemaMapper is not None: 

452 # finalize the schema 

453 self.schema = self.schemaMapper.getOutputSchema() 

454 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

455 

456 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

457 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

458 self.outputSchema = sourceCatSchema 

459 

460 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

461 inputs = butlerQC.get(inputRefs) 

462 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

463 

464 if self.config.doAstrometry: 

465 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

466 for ref in inputRefs.astromRefCat], 

467 refCats=inputs.pop('astromRefCat'), 

468 name=self.config.connections.astromRefCat, 

469 config=self.config.astromRefObjLoader, log=self.log) 

470 self.astrometry.setRefObjLoader(refObjLoader) 

471 

472 if self.config.doPhotoCal: 

473 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

474 for ref in inputRefs.photoRefCat], 

475 refCats=inputs.pop('photoRefCat'), 

476 name=self.config.connections.photoRefCat, 

477 config=self.config.photoRefObjLoader, 

478 log=self.log) 

479 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

480 

481 outputs = self.run(**inputs) 

482 

483 if self.config.doWriteMatches and self.config.doAstrometry: 

484 if outputs.astromMatches is not None: 

485 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

486 normalizedMatches.table.setMetadata(outputs.matchMeta) 

487 if self.config.doWriteMatchesDenormalized: 

488 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

489 outputs.matchesDenormalized = denormMatches 

490 outputs.matches = normalizedMatches 

491 else: 

492 del outputRefs.matches 

493 if self.config.doWriteMatchesDenormalized: 

494 del outputRefs.matchesDenormalized 

495 butlerQC.put(outputs, outputRefs) 

496 

497 @timeMethod 

498 def run(self, exposure, exposureIdInfo=None, background=None, 

499 icSourceCat=None, idGenerator=None): 

500 """Calibrate an exposure. 

501 

502 Parameters 

503 ---------- 

504 exposure : `lsst.afw.image.ExposureF` 

505 Exposure to calibrate. 

506 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

507 Exposure ID info. Deprecated in favor of ``idGenerator``, and 

508 ignored if that is provided. 

509 background : `lsst.afw.math.BackgroundList`, optional 

510 Initial model of background already subtracted from exposure. 

511 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

512 SourceCatalog from CharacterizeImageTask from which we can copy 

513 some fields. 

514 idGenerator : `lsst.meas.base.IdGenerator`, optional 

515 Object that generates source IDs and provides RNG seeds. 

516 

517 Returns 

518 ------- 

519 result : `lsst.pipe.base.Struct` 

520 Results as a struct with attributes: 

521 

522 ``exposure`` 

523 Characterized exposure (`lsst.afw.image.ExposureF`). 

524 ``sourceCat`` 

525 Detected sources (`lsst.afw.table.SourceCatalog`). 

526 ``outputBackground`` 

527 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

528 ``astromMatches`` 

529 List of source/ref matches from astrometry solver. 

530 ``matchMeta`` 

531 Metadata from astrometry matches. 

532 ``outputExposure`` 

533 Another reference to ``exposure`` for compatibility. 

534 ``outputCat`` 

535 Another reference to ``sourceCat`` for compatibility. 

536 """ 

537 # detect, deblend and measure sources 

538 if idGenerator is None: 

539 if exposureIdInfo is not None: 

540 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo) 

541 else: 

542 idGenerator = IdGenerator() 

543 

544 if background is None: 

545 background = BackgroundList() 

546 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

547 table.setMetadata(self.algMetadata) 

548 

549 detRes = self.detection.run(table=table, exposure=exposure, 

550 doSmooth=True) 

551 sourceCat = detRes.sources 

552 if detRes.background: 

553 for bg in detRes.background: 

554 background.append(bg) 

555 if self.config.doSkySources: 

556 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

557 if skySourceFootprints: 

558 for foot in skySourceFootprints: 

559 s = sourceCat.addNew() 

560 s.setFootprint(foot) 

561 s.set(self.skySourceKey, True) 

562 if self.config.doDeblend: 

563 self.deblend.run(exposure=exposure, sources=sourceCat) 

564 self.measurement.run( 

565 measCat=sourceCat, 

566 exposure=exposure, 

567 exposureId=idGenerator.catalog_id, 

568 ) 

569 if self.config.doApCorr: 

570 apCorrMap = exposure.getInfo().getApCorrMap() 

571 if apCorrMap is None: 

572 self.log.warning("Image does not have valid aperture correction map for %r; " 

573 "skipping aperture correction", idGenerator) 

574 else: 

575 self.applyApCorr.run( 

576 catalog=sourceCat, 

577 apCorrMap=apCorrMap, 

578 ) 

579 self.catalogCalculation.run(sourceCat) 

580 

581 self.setPrimaryFlags.run(sourceCat) 

582 

583 if icSourceCat is not None and \ 

584 len(self.config.icSourceFieldsToCopy) > 0: 

585 self.copyIcSourceFields(icSourceCat=icSourceCat, 

586 sourceCat=sourceCat) 

587 

588 # TODO DM-11568: this contiguous check-and-copy could go away if we 

589 # reserve enough space during SourceDetection and/or SourceDeblend. 

590 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

591 # contiguity now, so views are preserved from here on. 

592 if not sourceCat.isContiguous(): 

593 sourceCat = sourceCat.copy(deep=True) 

594 

595 # perform astrometry calibration: 

596 # fit an improved WCS and update the exposure's WCS in place 

597 astromMatches = None 

598 matchMeta = None 

599 if self.config.doAstrometry: 

600 astromRes = self.astrometry.run( 

601 exposure=exposure, 

602 sourceCat=sourceCat, 

603 ) 

604 astromMatches = astromRes.matches 

605 matchMeta = astromRes.matchMeta 

606 if exposure.getWcs() is None: 

607 if self.config.requireAstrometry: 

608 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

609 "is True.") 

610 else: 

611 self.log.warning("Unable to perform astrometric calibration for %r but " 

612 "requireAstrometry is False: attempting to proceed...", 

613 idGenerator) 

614 

615 # compute photometric calibration 

616 if self.config.doPhotoCal: 

617 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

618 if self.config.requirePhotoCal: 

619 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

620 "photoCal, but requirePhotoCal is True.") 

621 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

622 "is False, so skipping photometric calibration and setting photoCalib " 

623 "to None. Attempting to proceed...", idGenerator) 

624 exposure.setPhotoCalib(None) 

625 self.setMetadata(exposure=exposure, photoRes=None) 

626 else: 

627 try: 

628 photoRes = self.photoCal.run( 

629 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

630 ) 

631 exposure.setPhotoCalib(photoRes.photoCalib) 

632 # TODO: reword this to phrase it in terms of the 

633 # calibration factor? 

634 self.log.info("Photometric zero-point: %f", 

635 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

636 self.setMetadata(exposure=exposure, photoRes=photoRes) 

637 except Exception as e: 

638 if self.config.requirePhotoCal: 

639 raise 

640 self.log.warning("Unable to perform photometric calibration " 

641 "(%s): attempting to proceed", e) 

642 self.setMetadata(exposure=exposure, photoRes=None) 

643 

644 self.postCalibrationMeasurement.run( 

645 measCat=sourceCat, 

646 exposure=exposure, 

647 exposureId=idGenerator.catalog_id, 

648 ) 

649 

650 if self.config.doComputeSummaryStats: 

651 summary = self.computeSummaryStats.run(exposure=exposure, 

652 sources=sourceCat, 

653 background=background) 

654 exposure.getInfo().setSummaryStats(summary) 

655 

656 frame = getDebugFrame(self._display, "calibrate") 

657 if frame: 

658 displayAstrometry( 

659 sourceCat=sourceCat, 

660 exposure=exposure, 

661 matches=astromMatches, 

662 frame=frame, 

663 pause=False, 

664 ) 

665 

666 return pipeBase.Struct( 

667 sourceCat=sourceCat, 

668 astromMatches=astromMatches, 

669 matchMeta=matchMeta, 

670 outputExposure=exposure, 

671 outputCat=sourceCat, 

672 outputBackground=background, 

673 ) 

674 

675 def setMetadata(self, exposure, photoRes=None): 

676 """Set task and exposure metadata. 

677 

678 Logs a warning continues if needed data is missing. 

679 

680 Parameters 

681 ---------- 

682 exposure : `lsst.afw.image.ExposureF` 

683 Exposure to set metadata on. 

684 photoRes : `lsst.pipe.base.Struct`, optional 

685 Result of running photoCal task. 

686 """ 

687 if photoRes is None: 

688 return 

689 

690 metadata = exposure.getMetadata() 

691 

692 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

693 try: 

694 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

695 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

696 except Exception: 

697 self.log.warning("Could not set normalized MAGZERO in header: no " 

698 "exposure time") 

699 magZero = math.nan 

700 

701 try: 

702 metadata.set('MAGZERO', magZero) 

703 metadata.set('MAGZERO_RMS', photoRes.sigma) 

704 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

705 metadata.set('COLORTERM1', 0.0) 

706 metadata.set('COLORTERM2', 0.0) 

707 metadata.set('COLORTERM3', 0.0) 

708 except Exception as e: 

709 self.log.warning("Could not set exposure metadata: %s", e) 

710 

711 def copyIcSourceFields(self, icSourceCat, sourceCat): 

712 """Match sources in an icSourceCat and a sourceCat and copy fields. 

713 

714 The fields copied are those specified by 

715 ``config.icSourceFieldsToCopy``. 

716 

717 Parameters 

718 ---------- 

719 icSourceCat : `lsst.afw.table.SourceCatalog` 

720 Catalog from which to copy fields. 

721 sourceCat : `lsst.afw.table.SourceCatalog` 

722 Catalog to which to copy fields. 

723 

724 Raises 

725 ------ 

726 RuntimeError 

727 Raised if any of the following occur: 

728 - icSourceSchema and icSourceKeys are not specified. 

729 - icSourceCat and sourceCat are not specified. 

730 - icSourceFieldsToCopy is empty. 

731 """ 

732 if self.schemaMapper is None: 

733 raise RuntimeError("To copy icSource fields you must specify " 

734 "icSourceSchema and icSourceKeys when " 

735 "constructing this task") 

736 if icSourceCat is None or sourceCat is None: 

737 raise RuntimeError("icSourceCat and sourceCat must both be " 

738 "specified") 

739 if len(self.config.icSourceFieldsToCopy) == 0: 

740 self.log.warning("copyIcSourceFields doing nothing because " 

741 "icSourceFieldsToCopy is empty") 

742 return 

743 

744 mc = afwTable.MatchControl() 

745 mc.findOnlyClosest = False # return all matched objects 

746 matches = afwTable.matchXy(icSourceCat, sourceCat, 

747 self.config.matchRadiusPix, mc) 

748 if self.config.doDeblend: 

749 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

750 # if deblended, keep children 

751 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

752 

753 # Because we had to allow multiple matches to handle parents, we now 

754 # need to prune to the best matches 

755 # closest matches as a dict of icSourceCat source ID: 

756 # (icSourceCat source, sourceCat source, distance in pixels) 

757 bestMatches = {} 

758 for m0, m1, d in matches: 

759 id0 = m0.getId() 

760 match = bestMatches.get(id0) 

761 if match is None or d <= match[2]: 

762 bestMatches[id0] = (m0, m1, d) 

763 matches = list(bestMatches.values()) 

764 

765 # Check that no sourceCat sources are listed twice (we already know 

766 # that each match has a unique icSourceCat source ID, due to using 

767 # that ID as the key in bestMatches) 

768 numMatches = len(matches) 

769 numUniqueSources = len(set(m[1].getId() for m in matches)) 

770 if numUniqueSources != numMatches: 

771 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

772 "sources", numMatches, numUniqueSources) 

773 

774 self.log.info("Copying flags from icSourceCat to sourceCat for " 

775 "%d sources", numMatches) 

776 

777 # For each match: set the calibSourceKey flag and copy the desired 

778 # fields 

779 for icSrc, src, d in matches: 

780 src.setFlag(self.calibSourceKey, True) 

781 # src.assign copies the footprint from icSrc, which we don't want 

782 # (DM-407) 

783 # so set icSrc's footprint to src's footprint before src.assign, 

784 # then restore it 

785 icSrcFootprint = icSrc.getFootprint() 

786 try: 

787 icSrc.setFootprint(src.getFootprint()) 

788 src.assign(icSrc, self.schemaMapper) 

789 finally: 

790 icSrc.setFootprint(icSrcFootprint)