Coverage for python/lsst/pipe/tasks/calibrate.py: 21%

287 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-20 02:29 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23import warnings 

24 

25from lsstDebug import getDebugFrame 

26import lsst.pex.config as pexConfig 

27import lsst.pipe.base as pipeBase 

28import lsst.pipe.base.connectionTypes as cT 

29import lsst.afw.table as afwTable 

30from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

31from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

32from lsst.obs.base import ExposureIdInfo 

33import lsst.daf.base as dafBase 

34from lsst.afw.math import BackgroundList 

35from lsst.afw.table import SourceTable 

36from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

37from lsst.meas.base import (SingleFrameMeasurementTask, 

38 ApplyApCorrTask, 

39 CatalogCalculationTask) 

40from lsst.meas.deblender import SourceDeblendTask 

41from lsst.utils.timer import timeMethod 

42from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

43from .fakes import BaseFakeSourcesTask 

44from .photoCal import PhotoCalTask 

45from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

46 

47 

48__all__ = ["CalibrateConfig", "CalibrateTask"] 

49 

50 

51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

52 defaultTemplates={}): 

53 

54 icSourceSchema = cT.InitInput( 

55 doc="Schema produced by characterize image task, used to initialize this task", 

56 name="icSrc_schema", 

57 storageClass="SourceCatalog", 

58 ) 

59 

60 outputSchema = cT.InitOutput( 

61 doc="Schema after CalibrateTask has been initialized", 

62 name="src_schema", 

63 storageClass="SourceCatalog", 

64 ) 

65 

66 exposure = cT.Input( 

67 doc="Input image to calibrate", 

68 name="icExp", 

69 storageClass="ExposureF", 

70 dimensions=("instrument", "visit", "detector"), 

71 ) 

72 

73 background = cT.Input( 

74 doc="Backgrounds determined by characterize task", 

75 name="icExpBackground", 

76 storageClass="Background", 

77 dimensions=("instrument", "visit", "detector"), 

78 ) 

79 

80 icSourceCat = cT.Input( 

81 doc="Source catalog created by characterize task", 

82 name="icSrc", 

83 storageClass="SourceCatalog", 

84 dimensions=("instrument", "visit", "detector"), 

85 ) 

86 

87 astromRefCat = cT.PrerequisiteInput( 

88 doc="Reference catalog to use for astrometry", 

89 name="gaia_dr2_20200414", 

90 storageClass="SimpleCatalog", 

91 dimensions=("skypix",), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 photoRefCat = cT.PrerequisiteInput( 

97 doc="Reference catalog to use for photometric calibration", 

98 name="ps1_pv3_3pi_20170110", 

99 storageClass="SimpleCatalog", 

100 dimensions=("skypix",), 

101 deferLoad=True, 

102 multiple=True 

103 ) 

104 

105 outputExposure = cT.Output( 

106 doc="Exposure after running calibration task", 

107 name="calexp", 

108 storageClass="ExposureF", 

109 dimensions=("instrument", "visit", "detector"), 

110 ) 

111 

112 outputCat = cT.Output( 

113 doc="Source catalog produced in calibrate task", 

114 name="src", 

115 storageClass="SourceCatalog", 

116 dimensions=("instrument", "visit", "detector"), 

117 ) 

118 

119 outputBackground = cT.Output( 

120 doc="Background models estimated in calibration task", 

121 name="calexpBackground", 

122 storageClass="Background", 

123 dimensions=("instrument", "visit", "detector"), 

124 ) 

125 

126 matches = cT.Output( 

127 doc="Source/refObj matches from the astrometry solver", 

128 name="srcMatch", 

129 storageClass="Catalog", 

130 dimensions=("instrument", "visit", "detector"), 

131 ) 

132 

133 matchesDenormalized = cT.Output( 

134 doc="Denormalized matches from astrometry solver", 

135 name="srcMatchFull", 

136 storageClass="Catalog", 

137 dimensions=("instrument", "visit", "detector"), 

138 ) 

139 

140 def __init__(self, *, config=None): 

141 super().__init__(config=config) 

142 

143 if config.doAstrometry is False: 

144 self.prerequisiteInputs.remove("astromRefCat") 

145 if config.doPhotoCal is False: 

146 self.prerequisiteInputs.remove("photoRefCat") 

147 

148 if config.doWriteMatches is False or config.doAstrometry is False: 

149 self.outputs.remove("matches") 

150 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

151 self.outputs.remove("matchesDenormalized") 

152 

153 

154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

155 """Config for CalibrateTask""" 

156 doWrite = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Save calibration results?", 

160 ) 

161 doWriteHeavyFootprintsInSources = pexConfig.Field( 

162 dtype=bool, 

163 default=True, 

164 doc="Include HeavyFootprint data in source table? If false then heavy " 

165 "footprints are saved as normal footprints, which saves some space" 

166 ) 

167 doWriteMatches = pexConfig.Field( 

168 dtype=bool, 

169 default=True, 

170 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

171 ) 

172 doWriteMatchesDenormalized = pexConfig.Field( 

173 dtype=bool, 

174 default=False, 

175 doc=("Write reference matches in denormalized format? " 

176 "This format uses more disk space, but is more convenient to " 

177 "read. Ignored if doWriteMatches=False or doWrite=False."), 

178 ) 

179 doAstrometry = pexConfig.Field( 

180 dtype=bool, 

181 default=True, 

182 doc="Perform astrometric calibration?", 

183 ) 

184 astromRefObjLoader = pexConfig.ConfigField( 

185 dtype=LoadReferenceObjectsConfig, 

186 doc="reference object loader for astrometric calibration", 

187 ) 

188 photoRefObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 doc="reference object loader for photometric calibration", 

191 ) 

192 astrometry = pexConfig.ConfigurableField( 

193 target=AstrometryTask, 

194 doc="Perform astrometric calibration to refine the WCS", 

195 ) 

196 requireAstrometry = pexConfig.Field( 

197 dtype=bool, 

198 default=True, 

199 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

200 "false."), 

201 ) 

202 doPhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc="Perform phometric calibration?", 

206 ) 

207 requirePhotoCal = pexConfig.Field( 

208 dtype=bool, 

209 default=True, 

210 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

211 "false."), 

212 ) 

213 photoCal = pexConfig.ConfigurableField( 

214 target=PhotoCalTask, 

215 doc="Perform photometric calibration", 

216 ) 

217 icSourceFieldsToCopy = pexConfig.ListField( 

218 dtype=str, 

219 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

220 doc=("Fields to copy from the icSource catalog to the output catalog " 

221 "for matching sources Any missing fields will trigger a " 

222 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

223 ) 

224 matchRadiusPix = pexConfig.Field( 

225 dtype=float, 

226 default=3, 

227 doc=("Match radius for matching icSourceCat objects to sourceCat " 

228 "objects (pixels)"), 

229 ) 

230 checkUnitsParseStrict = pexConfig.Field( 

231 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

232 "'warn' or 'silent'"), 

233 dtype=str, 

234 default="raise", 

235 ) 

236 detection = pexConfig.ConfigurableField( 

237 target=SourceDetectionTask, 

238 doc="Detect sources" 

239 ) 

240 doDeblend = pexConfig.Field( 

241 dtype=bool, 

242 default=True, 

243 doc="Run deblender input exposure" 

244 ) 

245 deblend = pexConfig.ConfigurableField( 

246 target=SourceDeblendTask, 

247 doc="Split blended sources into their components" 

248 ) 

249 doSkySources = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Generate sky sources?", 

253 ) 

254 skySources = pexConfig.ConfigurableField( 

255 target=SkyObjectsTask, 

256 doc="Generate sky sources", 

257 ) 

258 measurement = pexConfig.ConfigurableField( 

259 target=SingleFrameMeasurementTask, 

260 doc="Measure sources" 

261 ) 

262 postCalibrationMeasurement = pexConfig.ConfigurableField( 

263 target=SingleFrameMeasurementTask, 

264 doc="Second round of measurement for plugins that need to be run after photocal" 

265 ) 

266 setPrimaryFlags = pexConfig.ConfigurableField( 

267 target=SetPrimaryFlagsTask, 

268 doc=("Set flags for primary source classification in single frame " 

269 "processing. True if sources are not sky sources and not a parent.") 

270 ) 

271 doApCorr = pexConfig.Field( 

272 dtype=bool, 

273 default=True, 

274 doc="Run subtask to apply aperture correction" 

275 ) 

276 applyApCorr = pexConfig.ConfigurableField( 

277 target=ApplyApCorrTask, 

278 doc="Subtask to apply aperture corrections" 

279 ) 

280 # If doApCorr is False, and the exposure does not have apcorrections 

281 # already applied, the active plugins in catalogCalculation almost 

282 # certainly should not contain the characterization plugin 

283 catalogCalculation = pexConfig.ConfigurableField( 

284 target=CatalogCalculationTask, 

285 doc="Subtask to run catalogCalculation plugins on catalog" 

286 ) 

287 doInsertFakes = pexConfig.Field( 

288 dtype=bool, 

289 default=False, 

290 doc="Run fake sources injection task", 

291 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. " 

292 "Please use ProcessCcdWithFakesTask instead.") 

293 ) 

294 insertFakes = pexConfig.ConfigurableField( 

295 target=BaseFakeSourcesTask, 

296 doc="Injection of fake sources for testing purposes (must be " 

297 "retargeted)", 

298 deprecated=("insertFakes is no longer supported. This config will be removed after v24. " 

299 "Please use ProcessCcdWithFakesTask instead.") 

300 ) 

301 doComputeSummaryStats = pexConfig.Field( 

302 dtype=bool, 

303 default=True, 

304 doc="Run subtask to measure exposure summary statistics?" 

305 ) 

306 computeSummaryStats = pexConfig.ConfigurableField( 

307 target=ComputeExposureSummaryStatsTask, 

308 doc="Subtask to run computeSummaryStats on exposure" 

309 ) 

310 doWriteExposure = pexConfig.Field( 

311 dtype=bool, 

312 default=True, 

313 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

314 "normal calexp but as a fakes_calexp." 

315 ) 

316 

317 def setDefaults(self): 

318 super().setDefaults() 

319 self.detection.doTempLocalBackground = False 

320 self.deblend.maxFootprintSize = 2000 

321 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

322 self.postCalibrationMeasurement.doReplaceWithNoise = False 

323 for key in self.postCalibrationMeasurement.slots: 

324 setattr(self.postCalibrationMeasurement.slots, key, None) 

325 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

326 # The photoRefCat connection is the name to use for the colorterms. 

327 self.photoCal.photoCatName = self.connections.photoRefCat 

328 

329 # NOTE: these two lines are for gen2, and are only here for compatibility. 

330 self.astromRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

331 self.photoRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

332 

333 def validate(self): 

334 super().validate() 

335 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

336 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

337 raise ValueError( 

338 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

339 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

340 ) 

341 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

342 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

343 raise ValueError( 

344 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

345 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

346 ) 

347 

348 

349class CalibrateTask(pipeBase.PipelineTask): 

350 """Task to calibrate an exposure. 

351 

352 Parameters 

353 ---------- 

354 butler : `None` 

355 Compatibility parameter. Should always be `None`. 

356 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

357 Reference object loader for astrometry task. Must be None if 

358 run as part of PipelineTask. 

359 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

360 Reference object loader for photometry task. Must be None if 

361 run as part of PipelineTask. 

362 icSourceSchema : `lsst.afw.table.Schema`, optional 

363 Schema for the icSource catalog. 

364 initInputs : `dict`, optional 

365 Dictionary that can contain a key ``icSourceSchema`` containing the 

366 input schema. If present will override the value of ``icSourceSchema``. 

367 """ 

368 ConfigClass = CalibrateConfig 

369 _DefaultName = "calibrate" 

370 

371 def __init__(self, butler=None, astromRefObjLoader=None, 

372 photoRefObjLoader=None, icSourceSchema=None, 

373 initInputs=None, **kwargs): 

374 super().__init__(**kwargs) 

375 

376 if butler is not None: 

377 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

378 category=FutureWarning, stacklevel=2) 

379 butler = None 

380 

381 if initInputs is not None: 

382 icSourceSchema = initInputs['icSourceSchema'].schema 

383 

384 if icSourceSchema is not None: 

385 # use a schema mapper to avoid copying each field separately 

386 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

387 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

388 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

389 

390 # Add fields to copy from an icSource catalog 

391 # and a field to indicate that the source matched a source in that 

392 # catalog. If any fields are missing then raise an exception, but 

393 # first find all missing fields in order to make the error message 

394 # more useful. 

395 self.calibSourceKey = self.schemaMapper.addOutputField( 

396 afwTable.Field["Flag"]("calib_detected", 

397 "Source was detected as an icSource")) 

398 missingFieldNames = [] 

399 for fieldName in self.config.icSourceFieldsToCopy: 

400 try: 

401 schemaItem = icSourceSchema.find(fieldName) 

402 except Exception: 

403 missingFieldNames.append(fieldName) 

404 else: 

405 # field found; if addMapping fails then raise an exception 

406 self.schemaMapper.addMapping(schemaItem.getKey()) 

407 

408 if missingFieldNames: 

409 raise RuntimeError("isSourceCat is missing fields {} " 

410 "specified in icSourceFieldsToCopy" 

411 .format(missingFieldNames)) 

412 

413 # produce a temporary schema to pass to the subtasks; finalize it 

414 # later 

415 self.schema = self.schemaMapper.editOutputSchema() 

416 else: 

417 self.schemaMapper = None 

418 self.schema = afwTable.SourceTable.makeMinimalSchema() 

419 self.makeSubtask('detection', schema=self.schema) 

420 

421 self.algMetadata = dafBase.PropertyList() 

422 

423 if self.config.doDeblend: 

424 self.makeSubtask("deblend", schema=self.schema) 

425 if self.config.doSkySources: 

426 self.makeSubtask("skySources") 

427 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

428 self.makeSubtask('measurement', schema=self.schema, 

429 algMetadata=self.algMetadata) 

430 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

431 algMetadata=self.algMetadata) 

432 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

433 if self.config.doApCorr: 

434 self.makeSubtask('applyApCorr', schema=self.schema) 

435 self.makeSubtask('catalogCalculation', schema=self.schema) 

436 

437 if self.config.doAstrometry: 

438 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

439 schema=self.schema) 

440 if self.config.doPhotoCal: 

441 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

442 schema=self.schema) 

443 if self.config.doComputeSummaryStats: 

444 self.makeSubtask('computeSummaryStats') 

445 

446 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

447 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

448 "reference object loaders.") 

449 

450 if self.schemaMapper is not None: 

451 # finalize the schema 

452 self.schema = self.schemaMapper.getOutputSchema() 

453 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

454 

455 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

456 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

457 self.outputSchema = sourceCatSchema 

458 

459 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

460 inputs = butlerQC.get(inputRefs) 

461 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

462 

463 if self.config.doAstrometry: 

464 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

465 for ref in inputRefs.astromRefCat], 

466 refCats=inputs.pop('astromRefCat'), 

467 config=self.config.astromRefObjLoader, log=self.log) 

468 self.astrometry.setRefObjLoader(refObjLoader) 

469 

470 if self.config.doPhotoCal: 

471 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

472 for ref in inputRefs.photoRefCat], 

473 refCats=inputs.pop('photoRefCat'), 

474 config=self.config.photoRefObjLoader, 

475 log=self.log) 

476 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

477 

478 outputs = self.run(**inputs) 

479 

480 if self.config.doWriteMatches and self.config.doAstrometry: 

481 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

482 normalizedMatches.table.setMetadata(outputs.matchMeta) 

483 if self.config.doWriteMatchesDenormalized: 

484 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

485 outputs.matchesDenormalized = denormMatches 

486 outputs.matches = normalizedMatches 

487 butlerQC.put(outputs, outputRefs) 

488 

489 @timeMethod 

490 def run(self, exposure, exposureIdInfo=None, background=None, 

491 icSourceCat=None): 

492 """Calibrate an exposure. 

493 

494 Parameters 

495 ---------- 

496 exposure : `lsst.afw.image.ExposureF` 

497 Exposure to calibrate. 

498 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

499 Exposure ID info. If not provided, returned SourceCatalog IDs will not 

500 be globally unique. 

501 background : `lsst.afw.math.BackgroundList`, optional 

502 Initial model of background already subtracted from exposure. 

503 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

504 SourceCatalog from CharacterizeImageTask from which we can copy some fields. 

505 

506 Returns 

507 ------- 

508 result : `lsst.pipe.base.Struct` 

509 Result structure with the following attributes: 

510 

511 ``exposure`` 

512 Characterized exposure (`lsst.afw.image.ExposureF`). 

513 ``sourceCat`` 

514 Detected sources (`lsst.afw.table.SourceCatalog`). 

515 ``outputBackground`` 

516 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

517 ``astromMatches`` 

518 List of source/ref matches from astrometry solver. 

519 ``matchMeta`` 

520 Metadata from astrometry matches. 

521 ``outputExposure`` 

522 Another reference to ``exposure`` for compatibility. 

523 ``outputCat`` 

524 Another reference to ``sourceCat`` for compatibility. 

525 """ 

526 # detect, deblend and measure sources 

527 if exposureIdInfo is None: 

528 exposureIdInfo = ExposureIdInfo() 

529 

530 if background is None: 

531 background = BackgroundList() 

532 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

533 table = SourceTable.make(self.schema, sourceIdFactory) 

534 table.setMetadata(self.algMetadata) 

535 

536 detRes = self.detection.run(table=table, exposure=exposure, 

537 doSmooth=True) 

538 sourceCat = detRes.sources 

539 if detRes.fpSets.background: 

540 for bg in detRes.fpSets.background: 

541 background.append(bg) 

542 if self.config.doSkySources: 

543 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

544 if skySourceFootprints: 

545 for foot in skySourceFootprints: 

546 s = sourceCat.addNew() 

547 s.setFootprint(foot) 

548 s.set(self.skySourceKey, True) 

549 if self.config.doDeblend: 

550 self.deblend.run(exposure=exposure, sources=sourceCat) 

551 self.measurement.run( 

552 measCat=sourceCat, 

553 exposure=exposure, 

554 exposureId=exposureIdInfo.expId 

555 ) 

556 if self.config.doApCorr: 

557 self.applyApCorr.run( 

558 catalog=sourceCat, 

559 apCorrMap=exposure.getInfo().getApCorrMap() 

560 ) 

561 self.catalogCalculation.run(sourceCat) 

562 

563 self.setPrimaryFlags.run(sourceCat) 

564 

565 if icSourceCat is not None and \ 

566 len(self.config.icSourceFieldsToCopy) > 0: 

567 self.copyIcSourceFields(icSourceCat=icSourceCat, 

568 sourceCat=sourceCat) 

569 

570 # TODO DM-11568: this contiguous check-and-copy could go away if we 

571 # reserve enough space during SourceDetection and/or SourceDeblend. 

572 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

573 # contiguity now, so views are preserved from here on. 

574 if not sourceCat.isContiguous(): 

575 sourceCat = sourceCat.copy(deep=True) 

576 

577 # perform astrometry calibration: 

578 # fit an improved WCS and update the exposure's WCS in place 

579 astromMatches = None 

580 matchMeta = None 

581 if self.config.doAstrometry: 

582 try: 

583 astromRes = self.astrometry.run( 

584 exposure=exposure, 

585 sourceCat=sourceCat, 

586 ) 

587 astromMatches = astromRes.matches 

588 matchMeta = astromRes.matchMeta 

589 except Exception as e: 

590 if self.config.requireAstrometry: 

591 raise 

592 self.log.warning("Unable to perform astrometric calibration " 

593 "(%s): attempting to proceed", e) 

594 

595 # compute photometric calibration 

596 if self.config.doPhotoCal: 

597 try: 

598 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

599 exposure.setPhotoCalib(photoRes.photoCalib) 

600 # TODO: reword this to phrase it in terms of the calibration factor? 

601 self.log.info("Photometric zero-point: %f", 

602 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

603 self.setMetadata(exposure=exposure, photoRes=photoRes) 

604 except Exception as e: 

605 if self.config.requirePhotoCal: 

606 raise 

607 self.log.warning("Unable to perform photometric calibration " 

608 "(%s): attempting to proceed", e) 

609 self.setMetadata(exposure=exposure, photoRes=None) 

610 

611 self.postCalibrationMeasurement.run( 

612 measCat=sourceCat, 

613 exposure=exposure, 

614 exposureId=exposureIdInfo.expId 

615 ) 

616 

617 if self.config.doComputeSummaryStats: 

618 summary = self.computeSummaryStats.run(exposure=exposure, 

619 sources=sourceCat, 

620 background=background) 

621 exposure.getInfo().setSummaryStats(summary) 

622 

623 frame = getDebugFrame(self._display, "calibrate") 

624 if frame: 

625 displayAstrometry( 

626 sourceCat=sourceCat, 

627 exposure=exposure, 

628 matches=astromMatches, 

629 frame=frame, 

630 pause=False, 

631 ) 

632 

633 return pipeBase.Struct( 

634 sourceCat=sourceCat, 

635 astromMatches=astromMatches, 

636 matchMeta=matchMeta, 

637 outputExposure=exposure, 

638 outputCat=sourceCat, 

639 outputBackground=background, 

640 ) 

641 

642 def getSchemaCatalogs(self): 

643 """Return a dict of empty catalogs for each catalog dataset produced 

644 by this task. 

645 """ 

646 sourceCat = afwTable.SourceCatalog(self.schema) 

647 sourceCat.getTable().setMetadata(self.algMetadata) 

648 return {"src": sourceCat} 

649 

650 def setMetadata(self, exposure, photoRes=None): 

651 """Set task and exposure metadata. 

652 

653 Logs a warning continues if needed data is missing. 

654 

655 Parameters 

656 ---------- 

657 exposure : `lsst.afw.image.ExposureF` 

658 Exposure to set metadata on. 

659 photoRes : `lsst.pipe.base.Struct`, optional 

660 Result of running photoCal task. 

661 """ 

662 if photoRes is None: 

663 return 

664 

665 metadata = exposure.getMetadata() 

666 

667 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

668 try: 

669 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

670 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

671 except Exception: 

672 self.log.warning("Could not set normalized MAGZERO in header: no " 

673 "exposure time") 

674 magZero = math.nan 

675 

676 try: 

677 metadata.set('MAGZERO', magZero) 

678 metadata.set('MAGZERO_RMS', photoRes.sigma) 

679 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

680 metadata.set('COLORTERM1', 0.0) 

681 metadata.set('COLORTERM2', 0.0) 

682 metadata.set('COLORTERM3', 0.0) 

683 except Exception as e: 

684 self.log.warning("Could not set exposure metadata: %s", e) 

685 

686 def copyIcSourceFields(self, icSourceCat, sourceCat): 

687 """Match sources in an icSourceCat and a sourceCat and copy fields. 

688 

689 The fields copied are those specified by ``config.icSourceFieldsToCopy``. 

690 

691 Parameters 

692 ---------- 

693 icSourceCat : `lsst.afw.table.SourceCatalog` 

694 Catalog from which to copy fields. 

695 sourceCat : `lsst.afw.table.SourceCatalog` 

696 Catalog to which to copy fields. 

697 """ 

698 if self.schemaMapper is None: 

699 raise RuntimeError("To copy icSource fields you must specify " 

700 "icSourceSchema nd icSourceKeys when " 

701 "constructing this task") 

702 if icSourceCat is None or sourceCat is None: 

703 raise RuntimeError("icSourceCat and sourceCat must both be " 

704 "specified") 

705 if len(self.config.icSourceFieldsToCopy) == 0: 

706 self.log.warning("copyIcSourceFields doing nothing because " 

707 "icSourceFieldsToCopy is empty") 

708 return 

709 

710 mc = afwTable.MatchControl() 

711 mc.findOnlyClosest = False # return all matched objects 

712 matches = afwTable.matchXy(icSourceCat, sourceCat, 

713 self.config.matchRadiusPix, mc) 

714 if self.config.doDeblend: 

715 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

716 # if deblended, keep children 

717 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

718 

719 # Because we had to allow multiple matches to handle parents, we now 

720 # need to prune to the best matches 

721 # closest matches as a dict of icSourceCat source ID: 

722 # (icSourceCat source, sourceCat source, distance in pixels) 

723 bestMatches = {} 

724 for m0, m1, d in matches: 

725 id0 = m0.getId() 

726 match = bestMatches.get(id0) 

727 if match is None or d <= match[2]: 

728 bestMatches[id0] = (m0, m1, d) 

729 matches = list(bestMatches.values()) 

730 

731 # Check that no sourceCat sources are listed twice (we already know 

732 # that each match has a unique icSourceCat source ID, due to using 

733 # that ID as the key in bestMatches) 

734 numMatches = len(matches) 

735 numUniqueSources = len(set(m[1].getId() for m in matches)) 

736 if numUniqueSources != numMatches: 

737 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

738 "sources", numMatches, numUniqueSources) 

739 

740 self.log.info("Copying flags from icSourceCat to sourceCat for " 

741 "%d sources", numMatches) 

742 

743 # For each match: set the calibSourceKey flag and copy the desired 

744 # fields 

745 for icSrc, src, d in matches: 

746 src.setFlag(self.calibSourceKey, True) 

747 # src.assign copies the footprint from icSrc, which we don't want 

748 # (DM-407) 

749 # so set icSrc's footprint to src's footprint before src.assign, 

750 # then restore it 

751 icSrcFootprint = icSrc.getFootprint() 

752 try: 

753 icSrc.setFootprint(src.getFootprint()) 

754 src.assign(icSrc, self.schemaMapper) 

755 finally: 

756 icSrc.setFootprint(icSrcFootprint)