Coverage for python/lsst/pipe/tasks/calibrate.py: 22%

277 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-30 10:45 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import warnings 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

33from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

34from lsst.obs.base import ExposureIdInfo 

35import lsst.daf.base as dafBase 

36from lsst.afw.math import BackgroundList 

37from lsst.afw.table import SourceTable 

38from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

39from lsst.meas.base import (SingleFrameMeasurementTask, 

40 ApplyApCorrTask, 

41 CatalogCalculationTask) 

42from lsst.meas.deblender import SourceDeblendTask 

43from lsst.utils.timer import timeMethod 

44from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

45from .fakes import BaseFakeSourcesTask 

46from .photoCal import PhotoCalTask 

47from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

48 

49 

50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

51 defaultTemplates={}): 

52 

53 icSourceSchema = cT.InitInput( 

54 doc="Schema produced by characterize image task, used to initialize this task", 

55 name="icSrc_schema", 

56 storageClass="SourceCatalog", 

57 ) 

58 

59 outputSchema = cT.InitOutput( 

60 doc="Schema after CalibrateTask has been initialized", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 exposure = cT.Input( 

66 doc="Input image to calibrate", 

67 name="icExp", 

68 storageClass="ExposureF", 

69 dimensions=("instrument", "visit", "detector"), 

70 ) 

71 

72 background = cT.Input( 

73 doc="Backgrounds determined by characterize task", 

74 name="icExpBackground", 

75 storageClass="Background", 

76 dimensions=("instrument", "visit", "detector"), 

77 ) 

78 

79 icSourceCat = cT.Input( 

80 doc="Source catalog created by characterize task", 

81 name="icSrc", 

82 storageClass="SourceCatalog", 

83 dimensions=("instrument", "visit", "detector"), 

84 ) 

85 

86 astromRefCat = cT.PrerequisiteInput( 

87 doc="Reference catalog to use for astrometry", 

88 name="gaia_dr2_20200414", 

89 storageClass="SimpleCatalog", 

90 dimensions=("skypix",), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 photoRefCat = cT.PrerequisiteInput( 

96 doc="Reference catalog to use for photometric calibration", 

97 name="ps1_pv3_3pi_20170110", 

98 storageClass="SimpleCatalog", 

99 dimensions=("skypix",), 

100 deferLoad=True, 

101 multiple=True 

102 ) 

103 

104 outputExposure = cT.Output( 

105 doc="Exposure after running calibration task", 

106 name="calexp", 

107 storageClass="ExposureF", 

108 dimensions=("instrument", "visit", "detector"), 

109 ) 

110 

111 outputCat = cT.Output( 

112 doc="Source catalog produced in calibrate task", 

113 name="src", 

114 storageClass="SourceCatalog", 

115 dimensions=("instrument", "visit", "detector"), 

116 ) 

117 

118 outputBackground = cT.Output( 

119 doc="Background models estimated in calibration task", 

120 name="calexpBackground", 

121 storageClass="Background", 

122 dimensions=("instrument", "visit", "detector"), 

123 ) 

124 

125 matches = cT.Output( 

126 doc="Source/refObj matches from the astrometry solver", 

127 name="srcMatch", 

128 storageClass="Catalog", 

129 dimensions=("instrument", "visit", "detector"), 

130 ) 

131 

132 matchesDenormalized = cT.Output( 

133 doc="Denormalized matches from astrometry solver", 

134 name="srcMatchFull", 

135 storageClass="Catalog", 

136 dimensions=("instrument", "visit", "detector"), 

137 ) 

138 

139 def __init__(self, *, config=None): 

140 super().__init__(config=config) 

141 

142 if config.doAstrometry is False: 

143 self.prerequisiteInputs.remove("astromRefCat") 

144 if config.doPhotoCal is False: 

145 self.prerequisiteInputs.remove("photoRefCat") 

146 

147 if config.doWriteMatches is False or config.doAstrometry is False: 

148 self.outputs.remove("matches") 

149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

150 self.outputs.remove("matchesDenormalized") 

151 

152 

153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

154 """Config for CalibrateTask.""" 

155 

156 doWrite = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Save calibration results?", 

160 ) 

161 doWriteHeavyFootprintsInSources = pexConfig.Field( 

162 dtype=bool, 

163 default=True, 

164 doc="Include HeavyFootprint data in source table? If false then heavy " 

165 "footprints are saved as normal footprints, which saves some space" 

166 ) 

167 doWriteMatches = pexConfig.Field( 

168 dtype=bool, 

169 default=True, 

170 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

171 ) 

172 doWriteMatchesDenormalized = pexConfig.Field( 

173 dtype=bool, 

174 default=False, 

175 doc=("Write reference matches in denormalized format? " 

176 "This format uses more disk space, but is more convenient to " 

177 "read. Ignored if doWriteMatches=False or doWrite=False."), 

178 ) 

179 doAstrometry = pexConfig.Field( 

180 dtype=bool, 

181 default=True, 

182 doc="Perform astrometric calibration?", 

183 ) 

184 astromRefObjLoader = pexConfig.ConfigField( 

185 dtype=LoadReferenceObjectsConfig, 

186 doc="reference object loader for astrometric calibration", 

187 ) 

188 photoRefObjLoader = pexConfig.ConfigField( 

189 dtype=LoadReferenceObjectsConfig, 

190 doc="reference object loader for photometric calibration", 

191 ) 

192 astrometry = pexConfig.ConfigurableField( 

193 target=AstrometryTask, 

194 doc="Perform astrometric calibration to refine the WCS", 

195 ) 

196 requireAstrometry = pexConfig.Field( 

197 dtype=bool, 

198 default=True, 

199 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

200 "false."), 

201 ) 

202 doPhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc="Perform phometric calibration?", 

206 ) 

207 requirePhotoCal = pexConfig.Field( 

208 dtype=bool, 

209 default=True, 

210 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

211 "false."), 

212 ) 

213 photoCal = pexConfig.ConfigurableField( 

214 target=PhotoCalTask, 

215 doc="Perform photometric calibration", 

216 ) 

217 icSourceFieldsToCopy = pexConfig.ListField( 

218 dtype=str, 

219 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

220 doc=("Fields to copy from the icSource catalog to the output catalog " 

221 "for matching sources Any missing fields will trigger a " 

222 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

223 ) 

224 matchRadiusPix = pexConfig.Field( 

225 dtype=float, 

226 default=3, 

227 doc=("Match radius for matching icSourceCat objects to sourceCat " 

228 "objects (pixels)"), 

229 ) 

230 checkUnitsParseStrict = pexConfig.Field( 

231 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

232 "'warn' or 'silent'"), 

233 dtype=str, 

234 default="raise", 

235 ) 

236 detection = pexConfig.ConfigurableField( 

237 target=SourceDetectionTask, 

238 doc="Detect sources" 

239 ) 

240 doDeblend = pexConfig.Field( 

241 dtype=bool, 

242 default=True, 

243 doc="Run deblender input exposure" 

244 ) 

245 deblend = pexConfig.ConfigurableField( 

246 target=SourceDeblendTask, 

247 doc="Split blended sources into their components" 

248 ) 

249 doSkySources = pexConfig.Field( 

250 dtype=bool, 

251 default=True, 

252 doc="Generate sky sources?", 

253 ) 

254 skySources = pexConfig.ConfigurableField( 

255 target=SkyObjectsTask, 

256 doc="Generate sky sources", 

257 ) 

258 measurement = pexConfig.ConfigurableField( 

259 target=SingleFrameMeasurementTask, 

260 doc="Measure sources" 

261 ) 

262 postCalibrationMeasurement = pexConfig.ConfigurableField( 

263 target=SingleFrameMeasurementTask, 

264 doc="Second round of measurement for plugins that need to be run after photocal" 

265 ) 

266 setPrimaryFlags = pexConfig.ConfigurableField( 

267 target=SetPrimaryFlagsTask, 

268 doc=("Set flags for primary source classification in single frame " 

269 "processing. True if sources are not sky sources and not a parent.") 

270 ) 

271 doApCorr = pexConfig.Field( 

272 dtype=bool, 

273 default=True, 

274 doc="Run subtask to apply aperture correction" 

275 ) 

276 applyApCorr = pexConfig.ConfigurableField( 

277 target=ApplyApCorrTask, 

278 doc="Subtask to apply aperture corrections" 

279 ) 

280 # If doApCorr is False, and the exposure does not have apcorrections 

281 # already applied, the active plugins in catalogCalculation almost 

282 # certainly should not contain the characterization plugin 

283 catalogCalculation = pexConfig.ConfigurableField( 

284 target=CatalogCalculationTask, 

285 doc="Subtask to run catalogCalculation plugins on catalog" 

286 ) 

287 doInsertFakes = pexConfig.Field( 

288 dtype=bool, 

289 default=False, 

290 doc="Run fake sources injection task", 

291 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. " 

292 "Please use ProcessCcdWithFakesTask instead.") 

293 ) 

294 insertFakes = pexConfig.ConfigurableField( 

295 target=BaseFakeSourcesTask, 

296 doc="Injection of fake sources for testing purposes (must be " 

297 "retargeted)", 

298 deprecated=("insertFakes is no longer supported. This config will be removed after v24. " 

299 "Please use ProcessCcdWithFakesTask instead.") 

300 ) 

301 doComputeSummaryStats = pexConfig.Field( 

302 dtype=bool, 

303 default=True, 

304 doc="Run subtask to measure exposure summary statistics?" 

305 ) 

306 computeSummaryStats = pexConfig.ConfigurableField( 

307 target=ComputeExposureSummaryStatsTask, 

308 doc="Subtask to run computeSummaryStats on exposure" 

309 ) 

310 doWriteExposure = pexConfig.Field( 

311 dtype=bool, 

312 default=True, 

313 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

314 "normal calexp but as a fakes_calexp." 

315 ) 

316 

317 def setDefaults(self): 

318 super().setDefaults() 

319 self.detection.doTempLocalBackground = False 

320 self.deblend.maxFootprintSize = 2000 

321 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

322 self.postCalibrationMeasurement.doReplaceWithNoise = False 

323 for key in self.postCalibrationMeasurement.slots: 

324 setattr(self.postCalibrationMeasurement.slots, key, None) 

325 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

326 # The photoRefCat connection is the name to use for the colorterms. 

327 self.photoCal.photoCatName = self.connections.photoRefCat 

328 

329 

330class CalibrateTask(pipeBase.PipelineTask): 

331 """Calibrate an exposure: measure sources and perform astrometric and 

332 photometric calibration. 

333 

334 Given an exposure with a good PSF model and aperture correction map(e.g. as 

335 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

336 perform the following operations: 

337 - Run detection and measurement 

338 - Run astrometry subtask to fit an improved WCS 

339 - Run photoCal subtask to fit the exposure's photometric zero-point 

340 

341 Parameters 

342 ---------- 

343 butler : `None` 

344 Compatibility parameter. Should always be `None`. 

345 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

346 Unused in gen3: must be `None`. 

347 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

348 Unused in gen3: must be `None`. 

349 icSourceSchema : `lsst.afw.table.Schema`, optional 

350 Schema for the icSource catalog. 

351 initInputs : `dict`, optional 

352 Dictionary that can contain a key ``icSourceSchema`` containing the 

353 input schema. If present will override the value of ``icSourceSchema``. 

354 

355 Raises 

356 ------ 

357 RuntimeError 

358 Raised if any of the following occur: 

359 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

360 - PipelineTask form of this task is initialized with reference object loaders. 

361 

362 Notes 

363 ----- 

364 Quantities set in exposure Metadata: 

365 

366 MAGZERO_RMS 

367 MAGZERO's RMS == sigma reported by photoCal task 

368 MAGZERO_NOBJ 

369 Number of stars used == ngood reported by photoCal task 

370 COLORTERM1 

371 ?? (always 0.0) 

372 COLORTERM2 

373 ?? (always 0.0) 

374 COLORTERM3 

375 ?? (always 0.0) 

376 

377 Debugging: 

378 CalibrateTask has a debug dictionary containing one key: 

379 

380 calibrate 

381 frame (an int; <= 0 to not display) in which to display the exposure, 

382 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

383 the meaning of the various symbols. 

384 """ 

385 

386 ConfigClass = CalibrateConfig 

387 _DefaultName = "calibrate" 

388 

389 def __init__(self, butler=None, astromRefObjLoader=None, 

390 photoRefObjLoader=None, icSourceSchema=None, 

391 initInputs=None, **kwargs): 

392 super().__init__(**kwargs) 

393 

394 if butler is not None: 

395 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.", 

396 category=FutureWarning, stacklevel=2) 

397 butler = None 

398 

399 if initInputs is not None: 

400 icSourceSchema = initInputs['icSourceSchema'].schema 

401 

402 if icSourceSchema is not None: 

403 # use a schema mapper to avoid copying each field separately 

404 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

405 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

406 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

407 

408 # Add fields to copy from an icSource catalog 

409 # and a field to indicate that the source matched a source in that 

410 # catalog. If any fields are missing then raise an exception, but 

411 # first find all missing fields in order to make the error message 

412 # more useful. 

413 self.calibSourceKey = self.schemaMapper.addOutputField( 

414 afwTable.Field["Flag"]("calib_detected", 

415 "Source was detected as an icSource")) 

416 missingFieldNames = [] 

417 for fieldName in self.config.icSourceFieldsToCopy: 

418 try: 

419 schemaItem = icSourceSchema.find(fieldName) 

420 except Exception: 

421 missingFieldNames.append(fieldName) 

422 else: 

423 # field found; if addMapping fails then raise an exception 

424 self.schemaMapper.addMapping(schemaItem.getKey()) 

425 

426 if missingFieldNames: 

427 raise RuntimeError("isSourceCat is missing fields {} " 

428 "specified in icSourceFieldsToCopy" 

429 .format(missingFieldNames)) 

430 

431 # produce a temporary schema to pass to the subtasks; finalize it 

432 # later 

433 self.schema = self.schemaMapper.editOutputSchema() 

434 else: 

435 self.schemaMapper = None 

436 self.schema = afwTable.SourceTable.makeMinimalSchema() 

437 self.makeSubtask('detection', schema=self.schema) 

438 

439 self.algMetadata = dafBase.PropertyList() 

440 

441 if self.config.doDeblend: 

442 self.makeSubtask("deblend", schema=self.schema) 

443 if self.config.doSkySources: 

444 self.makeSubtask("skySources") 

445 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

446 self.makeSubtask('measurement', schema=self.schema, 

447 algMetadata=self.algMetadata) 

448 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

449 algMetadata=self.algMetadata) 

450 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

451 if self.config.doApCorr: 

452 self.makeSubtask('applyApCorr', schema=self.schema) 

453 self.makeSubtask('catalogCalculation', schema=self.schema) 

454 

455 if self.config.doAstrometry: 

456 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

457 schema=self.schema) 

458 if self.config.doPhotoCal: 

459 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

460 schema=self.schema) 

461 if self.config.doComputeSummaryStats: 

462 self.makeSubtask('computeSummaryStats') 

463 

464 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

465 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

466 "reference object loaders.") 

467 

468 if self.schemaMapper is not None: 

469 # finalize the schema 

470 self.schema = self.schemaMapper.getOutputSchema() 

471 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

472 

473 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

474 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

475 self.outputSchema = sourceCatSchema 

476 

477 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

478 inputs = butlerQC.get(inputRefs) 

479 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

480 

481 if self.config.doAstrometry: 

482 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

483 for ref in inputRefs.astromRefCat], 

484 refCats=inputs.pop('astromRefCat'), 

485 name=self.config.connections.astromRefCat, 

486 config=self.config.astromRefObjLoader, log=self.log) 

487 self.astrometry.setRefObjLoader(refObjLoader) 

488 

489 if self.config.doPhotoCal: 

490 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

491 for ref in inputRefs.photoRefCat], 

492 refCats=inputs.pop('photoRefCat'), 

493 name=self.config.connections.photoRefCat, 

494 config=self.config.photoRefObjLoader, 

495 log=self.log) 

496 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

497 

498 outputs = self.run(**inputs) 

499 

500 if self.config.doWriteMatches and self.config.doAstrometry: 

501 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

502 normalizedMatches.table.setMetadata(outputs.matchMeta) 

503 if self.config.doWriteMatchesDenormalized: 

504 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

505 outputs.matchesDenormalized = denormMatches 

506 outputs.matches = normalizedMatches 

507 butlerQC.put(outputs, outputRefs) 

508 

509 @timeMethod 

510 def run(self, exposure, exposureIdInfo=None, background=None, 

511 icSourceCat=None): 

512 """Calibrate an exposure. 

513 

514 Parameters 

515 ---------- 

516 exposure : `lsst.afw.image.ExposureF` 

517 Exposure to calibrate. 

518 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional 

519 Exposure ID info. If not provided, returned SourceCatalog IDs will not 

520 be globally unique. 

521 background : `lsst.afw.math.BackgroundList`, optional 

522 Initial model of background already subtracted from exposure. 

523 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

524 SourceCatalog from CharacterizeImageTask from which we can copy some fields. 

525 

526 Returns 

527 ------- 

528 result : `lsst.pipe.base.Struct` 

529 Results as a struct with attributes: 

530 

531 ``exposure`` 

532 Characterized exposure (`lsst.afw.image.ExposureF`). 

533 ``sourceCat`` 

534 Detected sources (`lsst.afw.table.SourceCatalog`). 

535 ``outputBackground`` 

536 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

537 ``astromMatches`` 

538 List of source/ref matches from astrometry solver. 

539 ``matchMeta`` 

540 Metadata from astrometry matches. 

541 ``outputExposure`` 

542 Another reference to ``exposure`` for compatibility. 

543 ``outputCat`` 

544 Another reference to ``sourceCat`` for compatibility. 

545 """ 

546 # detect, deblend and measure sources 

547 if exposureIdInfo is None: 

548 exposureIdInfo = ExposureIdInfo() 

549 

550 if background is None: 

551 background = BackgroundList() 

552 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

553 table = SourceTable.make(self.schema, sourceIdFactory) 

554 table.setMetadata(self.algMetadata) 

555 

556 detRes = self.detection.run(table=table, exposure=exposure, 

557 doSmooth=True) 

558 sourceCat = detRes.sources 

559 if detRes.fpSets.background: 

560 for bg in detRes.fpSets.background: 

561 background.append(bg) 

562 if self.config.doSkySources: 

563 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

564 if skySourceFootprints: 

565 for foot in skySourceFootprints: 

566 s = sourceCat.addNew() 

567 s.setFootprint(foot) 

568 s.set(self.skySourceKey, True) 

569 if self.config.doDeblend: 

570 self.deblend.run(exposure=exposure, sources=sourceCat) 

571 self.measurement.run( 

572 measCat=sourceCat, 

573 exposure=exposure, 

574 exposureId=exposureIdInfo.expId 

575 ) 

576 if self.config.doApCorr: 

577 self.applyApCorr.run( 

578 catalog=sourceCat, 

579 apCorrMap=exposure.getInfo().getApCorrMap() 

580 ) 

581 self.catalogCalculation.run(sourceCat) 

582 

583 self.setPrimaryFlags.run(sourceCat) 

584 

585 if icSourceCat is not None and \ 

586 len(self.config.icSourceFieldsToCopy) > 0: 

587 self.copyIcSourceFields(icSourceCat=icSourceCat, 

588 sourceCat=sourceCat) 

589 

590 # TODO DM-11568: this contiguous check-and-copy could go away if we 

591 # reserve enough space during SourceDetection and/or SourceDeblend. 

592 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

593 # contiguity now, so views are preserved from here on. 

594 if not sourceCat.isContiguous(): 

595 sourceCat = sourceCat.copy(deep=True) 

596 

597 # perform astrometry calibration: 

598 # fit an improved WCS and update the exposure's WCS in place 

599 astromMatches = None 

600 matchMeta = None 

601 if self.config.doAstrometry: 

602 try: 

603 astromRes = self.astrometry.run( 

604 exposure=exposure, 

605 sourceCat=sourceCat, 

606 ) 

607 astromMatches = astromRes.matches 

608 matchMeta = astromRes.matchMeta 

609 except Exception as e: 

610 if self.config.requireAstrometry: 

611 raise 

612 self.log.warning("Unable to perform astrometric calibration " 

613 "(%s): attempting to proceed", e) 

614 

615 # compute photometric calibration 

616 if self.config.doPhotoCal: 

617 try: 

618 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

619 exposure.setPhotoCalib(photoRes.photoCalib) 

620 # TODO: reword this to phrase it in terms of the calibration factor? 

621 self.log.info("Photometric zero-point: %f", 

622 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

623 self.setMetadata(exposure=exposure, photoRes=photoRes) 

624 except Exception as e: 

625 if self.config.requirePhotoCal: 

626 raise 

627 self.log.warning("Unable to perform photometric calibration " 

628 "(%s): attempting to proceed", e) 

629 self.setMetadata(exposure=exposure, photoRes=None) 

630 

631 self.postCalibrationMeasurement.run( 

632 measCat=sourceCat, 

633 exposure=exposure, 

634 exposureId=exposureIdInfo.expId 

635 ) 

636 

637 if self.config.doComputeSummaryStats: 

638 summary = self.computeSummaryStats.run(exposure=exposure, 

639 sources=sourceCat, 

640 background=background) 

641 exposure.getInfo().setSummaryStats(summary) 

642 

643 frame = getDebugFrame(self._display, "calibrate") 

644 if frame: 

645 displayAstrometry( 

646 sourceCat=sourceCat, 

647 exposure=exposure, 

648 matches=astromMatches, 

649 frame=frame, 

650 pause=False, 

651 ) 

652 

653 return pipeBase.Struct( 

654 sourceCat=sourceCat, 

655 astromMatches=astromMatches, 

656 matchMeta=matchMeta, 

657 outputExposure=exposure, 

658 outputCat=sourceCat, 

659 outputBackground=background, 

660 ) 

661 

662 def getSchemaCatalogs(self): 

663 """Return a dict of empty catalogs for each catalog dataset produced 

664 by this task. 

665 """ 

666 sourceCat = afwTable.SourceCatalog(self.schema) 

667 sourceCat.getTable().setMetadata(self.algMetadata) 

668 return {"src": sourceCat} 

669 

670 def setMetadata(self, exposure, photoRes=None): 

671 """Set task and exposure metadata. 

672 

673 Logs a warning continues if needed data is missing. 

674 

675 Parameters 

676 ---------- 

677 exposure : `lsst.afw.image.ExposureF` 

678 Exposure to set metadata on. 

679 photoRes : `lsst.pipe.base.Struct`, optional 

680 Result of running photoCal task. 

681 """ 

682 if photoRes is None: 

683 return 

684 

685 metadata = exposure.getMetadata() 

686 

687 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

688 try: 

689 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

690 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

691 except Exception: 

692 self.log.warning("Could not set normalized MAGZERO in header: no " 

693 "exposure time") 

694 magZero = math.nan 

695 

696 try: 

697 metadata.set('MAGZERO', magZero) 

698 metadata.set('MAGZERO_RMS', photoRes.sigma) 

699 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

700 metadata.set('COLORTERM1', 0.0) 

701 metadata.set('COLORTERM2', 0.0) 

702 metadata.set('COLORTERM3', 0.0) 

703 except Exception as e: 

704 self.log.warning("Could not set exposure metadata: %s", e) 

705 

706 def copyIcSourceFields(self, icSourceCat, sourceCat): 

707 """Match sources in an icSourceCat and a sourceCat and copy fields. 

708 

709 The fields copied are those specified by ``config.icSourceFieldsToCopy``. 

710 

711 Parameters 

712 ---------- 

713 icSourceCat : `lsst.afw.table.SourceCatalog` 

714 Catalog from which to copy fields. 

715 sourceCat : `lsst.afw.table.SourceCatalog` 

716 Catalog to which to copy fields. 

717 

718 Raises 

719 ------ 

720 RuntimeError 

721 Raised if any of the following occur: 

722 - icSourceSchema and icSourceKeys are not specified. 

723 - icSourceCat and sourceCat are not specified. 

724 - icSourceFieldsToCopy is empty. 

725 """ 

726 if self.schemaMapper is None: 

727 raise RuntimeError("To copy icSource fields you must specify " 

728 "icSourceSchema and icSourceKeys when " 

729 "constructing this task") 

730 if icSourceCat is None or sourceCat is None: 

731 raise RuntimeError("icSourceCat and sourceCat must both be " 

732 "specified") 

733 if len(self.config.icSourceFieldsToCopy) == 0: 

734 self.log.warning("copyIcSourceFields doing nothing because " 

735 "icSourceFieldsToCopy is empty") 

736 return 

737 

738 mc = afwTable.MatchControl() 

739 mc.findOnlyClosest = False # return all matched objects 

740 matches = afwTable.matchXy(icSourceCat, sourceCat, 

741 self.config.matchRadiusPix, mc) 

742 if self.config.doDeblend: 

743 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

744 # if deblended, keep children 

745 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

746 

747 # Because we had to allow multiple matches to handle parents, we now 

748 # need to prune to the best matches 

749 # closest matches as a dict of icSourceCat source ID: 

750 # (icSourceCat source, sourceCat source, distance in pixels) 

751 bestMatches = {} 

752 for m0, m1, d in matches: 

753 id0 = m0.getId() 

754 match = bestMatches.get(id0) 

755 if match is None or d <= match[2]: 

756 bestMatches[id0] = (m0, m1, d) 

757 matches = list(bestMatches.values()) 

758 

759 # Check that no sourceCat sources are listed twice (we already know 

760 # that each match has a unique icSourceCat source ID, due to using 

761 # that ID as the key in bestMatches) 

762 numMatches = len(matches) 

763 numUniqueSources = len(set(m[1].getId() for m in matches)) 

764 if numUniqueSources != numMatches: 

765 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

766 "sources", numMatches, numUniqueSources) 

767 

768 self.log.info("Copying flags from icSourceCat to sourceCat for " 

769 "%d sources", numMatches) 

770 

771 # For each match: set the calibSourceKey flag and copy the desired 

772 # fields 

773 for icSrc, src, d in matches: 

774 src.setFlag(self.calibSourceKey, True) 

775 # src.assign copies the footprint from icSrc, which we don't want 

776 # (DM-407) 

777 # so set icSrc's footprint to src's footprint before src.assign, 

778 # then restore it 

779 icSrcFootprint = icSrc.getFootprint() 

780 try: 

781 icSrc.setFootprint(src.getFootprint()) 

782 src.assign(icSrc, self.schemaMapper) 

783 finally: 

784 icSrc.setFootprint(icSrcFootprint)