Coverage for python / lsst / pipe / tasks / calibrate.py: 17%

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["CalibrateConfig", "CalibrateTask"] 

23 

24import math 

25import numpy as np 

26 

27from lsstDebug import getDebugFrame 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.afw.table as afwTable 

32from lsst.ip.diffim.utils import evaluateMaskFraction 

33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches, AstrometryError 

34from lsst.meas.algorithms import LoadReferenceObjectsConfig, SkyObjectsTask 

35import lsst.daf.base as dafBase 

36from lsst.afw.math import BackgroundList 

37from lsst.afw.table import SourceTable 

38from lsst.meas.algorithms import (SourceDetectionTask, 

39 ReferenceObjectLoader, 

40 SetPrimaryFlagsTask, 

41 NormalizedCalibrationFluxTask) 

42from lsst.meas.base import (SingleFrameMeasurementTask, 

43 ApplyApCorrTask, 

44 CatalogCalculationTask, 

45 IdGenerator, 

46 DetectorVisitIdGeneratorConfig) 

47from lsst.meas.deblender import SourceDeblendTask 

48from lsst.utils.timer import timeMethod 

49from .photoCal import PhotoCalTask 

50from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

51 

52 

53class _EmptyTargetTask(pipeBase.PipelineTask): 

54 """ 

55 This is a placeholder target for CreateSummaryMetrics and must be retargeted at runtime. 

56 CreateSummaryMetrics should target an analysis tool task, but that would, at the time 

57 of writing, result in a circular import. 

58 

59 As a result, this class should not be used for anything else. 

60 """ 

61 ConfigClass = pipeBase.PipelineTaskConfig 

62 

63 def __init__(self, **kwargs) -> None: 

64 raise NotImplementedError( 

65 "doCreateSummaryMetrics is set to True, in which case " 

66 "createSummaryMetrics must be retargeted." 

67 ) 

68 

69 

70class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

71 defaultTemplates={}): 

72 

73 icSourceSchema = cT.InitInput( 

74 doc="Schema produced by characterize image task, used to initialize this task", 

75 name="icSrc_schema", 

76 storageClass="SourceCatalog", 

77 ) 

78 

79 outputSchema = cT.InitOutput( 

80 doc="Schema after CalibrateTask has been initialized", 

81 name="src_schema", 

82 storageClass="SourceCatalog", 

83 ) 

84 

85 exposure = cT.Input( 

86 doc="Input image to calibrate", 

87 name="icExp", 

88 storageClass="ExposureF", 

89 dimensions=("instrument", "visit", "detector"), 

90 ) 

91 

92 background = cT.Input( 

93 doc="Backgrounds determined by characterize task", 

94 name="icExpBackground", 

95 storageClass="Background", 

96 dimensions=("instrument", "visit", "detector"), 

97 ) 

98 

99 icSourceCat = cT.Input( 

100 doc="Source catalog created by characterize task", 

101 name="icSrc", 

102 storageClass="SourceCatalog", 

103 dimensions=("instrument", "visit", "detector"), 

104 ) 

105 

106 astromRefCat = cT.PrerequisiteInput( 

107 doc="Reference catalog to use for astrometry", 

108 name="gaia_dr3_20230707", 

109 storageClass="SimpleCatalog", 

110 dimensions=("skypix",), 

111 deferLoad=True, 

112 multiple=True, 

113 ) 

114 

115 photoRefCat = cT.PrerequisiteInput( 

116 doc="Reference catalog to use for photometric calibration", 

117 name="ps1_pv3_3pi_20170110", 

118 storageClass="SimpleCatalog", 

119 dimensions=("skypix",), 

120 deferLoad=True, 

121 multiple=True 

122 ) 

123 

124 outputExposure = cT.Output( 

125 doc="Exposure after running calibration task", 

126 name="calexp", 

127 storageClass="ExposureF", 

128 dimensions=("instrument", "visit", "detector"), 

129 ) 

130 

131 outputCat = cT.Output( 

132 doc="Source catalog produced in calibrate task", 

133 name="src", 

134 storageClass="SourceCatalog", 

135 dimensions=("instrument", "visit", "detector"), 

136 ) 

137 

138 outputBackground = cT.Output( 

139 doc="Background models estimated in calibration task", 

140 name="calexpBackground", 

141 storageClass="Background", 

142 dimensions=("instrument", "visit", "detector"), 

143 ) 

144 

145 outputSummaryMetrics = cT.Output( 

146 doc="Summary metrics created by the calibration task", 

147 name="calexpSummary_metrics", 

148 storageClass="MetricMeasurementBundle", 

149 dimensions=("instrument", "visit", "detector"), 

150 ) 

151 

152 matches = cT.Output( 

153 doc="Source/refObj matches from the astrometry solver", 

154 name="srcMatch", 

155 storageClass="Catalog", 

156 dimensions=("instrument", "visit", "detector"), 

157 ) 

158 

159 matchesDenormalized = cT.Output( 

160 doc="Denormalized matches from astrometry solver", 

161 name="srcMatchFull", 

162 storageClass="Catalog", 

163 dimensions=("instrument", "visit", "detector"), 

164 ) 

165 

166 def __init__(self, *, config=None): 

167 super().__init__(config=config) 

168 

169 if config.doAstrometry is False: 

170 self.prerequisiteInputs.remove("astromRefCat") 

171 if config.doPhotoCal is False: 

172 self.prerequisiteInputs.remove("photoRefCat") 

173 

174 if config.doWriteMatches is False or config.doAstrometry is False: 

175 self.outputs.remove("matches") 

176 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

177 self.outputs.remove("matchesDenormalized") 

178 

179 if config.doCreateSummaryMetrics is False: 

180 self.outputs.remove("outputSummaryMetrics") 

181 

182 

183class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

184 """Config for CalibrateTask.""" 

185 

186 doWrite = pexConfig.Field( 

187 dtype=bool, 

188 default=True, 

189 doc="Save calibration results?", 

190 ) 

191 doWriteHeavyFootprintsInSources = pexConfig.Field( 

192 dtype=bool, 

193 default=True, 

194 doc="Include HeavyFootprint data in source table? If false then heavy " 

195 "footprints are saved as normal footprints, which saves some space" 

196 ) 

197 doWriteMatches = pexConfig.Field( 

198 dtype=bool, 

199 default=True, 

200 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

201 ) 

202 doWriteMatchesDenormalized = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc=("Write reference matches in denormalized format? " 

206 "This format uses more disk space, but is more convenient to " 

207 "read for debugging. Ignored if doWriteMatches=False or doWrite=False."), 

208 ) 

209 doAstrometry = pexConfig.Field( 

210 dtype=bool, 

211 default=True, 

212 doc="Perform astrometric calibration?", 

213 ) 

214 astromRefObjLoader = pexConfig.ConfigField( 

215 dtype=LoadReferenceObjectsConfig, 

216 doc="reference object loader for astrometric calibration", 

217 ) 

218 photoRefObjLoader = pexConfig.ConfigField( 

219 dtype=LoadReferenceObjectsConfig, 

220 doc="reference object loader for photometric calibration", 

221 ) 

222 astrometry = pexConfig.ConfigurableField( 

223 target=AstrometryTask, 

224 doc="Perform astrometric calibration to refine the WCS", 

225 ) 

226 astrometryDetection = pexConfig.ConfigurableField( 

227 target=SourceDetectionTask, 

228 doc="Task to detect sources to used in the astrometric fit." 

229 ) 

230 

231 requireAstrometry = pexConfig.Field( 

232 dtype=bool, 

233 default=True, 

234 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

235 "false."), 

236 ) 

237 doPhotoCal = pexConfig.Field( 

238 dtype=bool, 

239 default=True, 

240 doc="Perform phometric calibration?", 

241 ) 

242 requirePhotoCal = pexConfig.Field( 

243 dtype=bool, 

244 default=True, 

245 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

246 "false."), 

247 ) 

248 photoCal = pexConfig.ConfigurableField( 

249 target=PhotoCalTask, 

250 doc="Perform photometric calibration", 

251 ) 

252 icSourceFieldsToCopy = pexConfig.ListField( 

253 dtype=str, 

254 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

255 doc=("Fields to copy from the icSource catalog to the output catalog " 

256 "for matching sources. Any missing fields will trigger a " 

257 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

258 ) 

259 astromFieldsToCopy = pexConfig.ListField( 

260 dtype=str, 

261 default=("calib_astrometry_used", ), 

262 doc=("Fields to copy from the astromCat catalog to the output catalog " 

263 "for matching sources. Any missing fields will trigger a " 

264 "RuntimeError exception. Ignored if astromCat does not exists.") 

265 ) 

266 

267 matchRadiusPix = pexConfig.Field( 

268 dtype=float, 

269 default=3, 

270 doc=("Match radius for matching icSourceCat objects to sourceCat " 

271 "objects (pixels)"), 

272 ) 

273 checkUnitsParseStrict = pexConfig.Field( 

274 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

275 "'warn' or 'silent'"), 

276 dtype=str, 

277 default="raise", 

278 ) 

279 detection = pexConfig.ConfigurableField( 

280 target=SourceDetectionTask, 

281 doc="Detect sources" 

282 ) 

283 doDeblend = pexConfig.Field( 

284 dtype=bool, 

285 default=True, 

286 doc="Run deblender input exposure" 

287 ) 

288 deblend = pexConfig.ConfigurableField( 

289 target=SourceDeblendTask, 

290 doc="Split blended sources into their components" 

291 ) 

292 doSkySources = pexConfig.Field( 

293 dtype=bool, 

294 default=True, 

295 doc="Generate sky sources?", 

296 ) 

297 skySources = pexConfig.ConfigurableField( 

298 target=SkyObjectsTask, 

299 doc="Generate sky sources", 

300 ) 

301 measurement = pexConfig.ConfigurableField( 

302 target=SingleFrameMeasurementTask, 

303 doc="Measure sources" 

304 ) 

305 doNormalizedCalibration = pexConfig.Field( 

306 dtype=bool, 

307 default=True, 

308 doc="Use normalized calibration flux (e.g. compensated tophats)?", 

309 ) 

310 normalizedCalibrationFlux = pexConfig.ConfigurableField( 

311 target=NormalizedCalibrationFluxTask, 

312 doc="Task to normalize the calibration flux (e.g. compensated tophats).", 

313 ) 

314 postCalibrationMeasurement = pexConfig.ConfigurableField( 

315 target=SingleFrameMeasurementTask, 

316 doc="Second round of measurement for plugins that need to be run after photocal" 

317 ) 

318 setPrimaryFlags = pexConfig.ConfigurableField( 

319 target=SetPrimaryFlagsTask, 

320 doc=("Set flags for primary source classification in single frame " 

321 "processing. True if sources are not sky sources and not a parent.") 

322 ) 

323 doApCorr = pexConfig.Field( 

324 dtype=bool, 

325 default=True, 

326 doc="Run subtask to apply aperture correction" 

327 ) 

328 applyApCorr = pexConfig.ConfigurableField( 

329 target=ApplyApCorrTask, 

330 doc="Subtask to apply aperture corrections" 

331 ) 

332 # If doApCorr is False, and the exposure does not have apcorrections 

333 # already applied, the active plugins in catalogCalculation almost 

334 # certainly should not contain the characterization plugin 

335 catalogCalculation = pexConfig.ConfigurableField( 

336 target=CatalogCalculationTask, 

337 doc="Subtask to run catalogCalculation plugins on catalog" 

338 ) 

339 doComputeSummaryStats = pexConfig.Field( 

340 dtype=bool, 

341 default=True, 

342 doc="Run subtask to measure exposure summary statistics?" 

343 ) 

344 computeSummaryStats = pexConfig.ConfigurableField( 

345 target=ComputeExposureSummaryStatsTask, 

346 doc="Subtask to run computeSummaryStats on exposure" 

347 ) 

348 doCreateSummaryMetrics = pexConfig.Field( 

349 dtype=bool, 

350 default=False, 

351 doc="Run the subtask to create summary metrics, and then write those metrics." 

352 ) 

353 createSummaryMetrics = pexConfig.ConfigurableField( 

354 target=_EmptyTargetTask, 

355 doc="Subtask to create metrics from the summary stats. This must be retargeted, likely to an" 

356 "analysis_tools task such as CalexpSummaryMetrics." 

357 ) 

358 doWriteExposure = pexConfig.Field( 

359 dtype=bool, 

360 default=True, 

361 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

362 "normal calexp but as a fakes_calexp." 

363 ) 

364 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

365 

366 def setDefaults(self): 

367 super().setDefaults() 

368 # Higher S/N detection pass for astrometry source selection 

369 self.astrometryDetection.thresholdValue = 50.0 

370 self.astrometryDetection.reEstimateBackground = False 

371 self.measurement.plugins.names |= ["base_CompensatedTophatFlux"] 

372 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] 

373 self.postCalibrationMeasurement.doReplaceWithNoise = False 

374 for key in self.postCalibrationMeasurement.slots: 

375 setattr(self.postCalibrationMeasurement.slots, key, None) 

376 self.astromRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

377 # The photoRefCat connection is the name to use for the colorterms. 

378 self.photoCal.photoCatName = self.connections.photoRefCat 

379 

380 self.normalizedCalibrationFlux.do_measure_ap_corr = False 

381 self.measurement.algorithms["base_CompensatedTophatFlux"].apertures = [12] 

382 

383 # TODO: Remove in DM-44658, streak masking to happen only in ip_diffim 

384 # Keep track of which footprints contain streaks 

385 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] 

386 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] 

387 

388 

389class CalibrateTask(pipeBase.PipelineTask): 

390 """Calibrate an exposure: measure sources and perform astrometric and 

391 photometric calibration. 

392 

393 Given an exposure with a good PSF model and aperture correction map(e.g. as 

394 provided by `~lsst.pipe.tasks.characterizeImage.CharacterizeImageTask`), 

395 perform the following operations: 

396 - Run detection and measurement 

397 - Run astrometry subtask to fit an improved WCS 

398 - Run photoCal subtask to fit the exposure's photometric zero-point 

399 

400 Parameters 

401 ---------- 

402 butler : `None` 

403 Compatibility parameter. Should always be `None`. 

404 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

405 Unused in gen3: must be `None`. 

406 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

407 Unused in gen3: must be `None`. 

408 icSourceSchema : `lsst.afw.table.Schema`, optional 

409 Schema for the icSource catalog. 

410 initInputs : `dict`, optional 

411 Dictionary that can contain a key ``icSourceSchema`` containing the 

412 input schema. If present will override the value of ``icSourceSchema``. 

413 

414 Raises 

415 ------ 

416 RuntimeError 

417 Raised if any of the following occur: 

418 - isSourceCat is missing fields specified in icSourceFieldsToCopy. 

419 - PipelineTask form of this task is initialized with reference object 

420 loaders. 

421 

422 Notes 

423 ----- 

424 Quantities set in exposure Metadata: 

425 

426 MAGZERO_RMS 

427 MAGZERO's RMS == sigma reported by photoCal task 

428 MAGZERO_NOBJ 

429 Number of stars used == ngood reported by photoCal task 

430 COLORTERM1 

431 ?? (always 0.0) 

432 COLORTERM2 

433 ?? (always 0.0) 

434 COLORTERM3 

435 ?? (always 0.0) 

436 

437 Debugging: 

438 CalibrateTask has a debug dictionary containing one key: 

439 

440 calibrate 

441 frame (an int; <= 0 to not display) in which to display the exposure, 

442 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

443 the meaning of the various symbols. 

444 """ 

445 

446 ConfigClass = CalibrateConfig 

447 _DefaultName = "calibrate" 

448 

449 def __init__(self, astromRefObjLoader=None, 

450 photoRefObjLoader=None, icSourceSchema=None, 

451 initInputs=None, **kwargs): 

452 super().__init__(**kwargs) 

453 

454 if initInputs is not None: 

455 icSourceSchema = initInputs['icSourceSchema'].schema 

456 

457 if icSourceSchema is not None: 

458 # use a schema mapper to avoid copying each field separately 

459 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

460 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

461 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

462 

463 # Add fields to copy from an icSource catalog 

464 # and a field to indicate that the source matched a source in that 

465 # catalog. If any fields are missing then raise an exception, but 

466 # first find all missing fields in order to make the error message 

467 # more useful. 

468 self.calibSourceKey = self.schemaMapper.addOutputField( 

469 afwTable.Field["Flag"]("calib_detected", 

470 "Source was detected as an icSource")) 

471 missingFieldNames = [] 

472 for fieldName in self.config.icSourceFieldsToCopy: 

473 try: 

474 schemaItem = icSourceSchema.find(fieldName) 

475 except Exception: 

476 missingFieldNames.append(fieldName) 

477 else: 

478 # field found; if addMapping fails then raise an exception 

479 self.schemaMapper.addMapping(schemaItem.getKey()) 

480 

481 if missingFieldNames: 

482 raise RuntimeError("isSourceCat is missing fields {} " 

483 "specified in icSourceFieldsToCopy" 

484 .format(missingFieldNames)) 

485 

486 # produce a temporary schema to pass to the subtasks; finalize it 

487 # later 

488 self.schema = self.schemaMapper.editOutputSchema() 

489 else: 

490 self.schemaMapper = None 

491 self.schema = afwTable.SourceTable.makeMinimalSchema() 

492 afwTable.CoordKey.addErrorFields(self.schema) 

493 self.makeSubtask('detection', schema=self.schema) 

494 

495 self.algMetadata = dafBase.PropertyList() 

496 

497 if self.config.doDeblend: 

498 self.makeSubtask("deblend", schema=self.schema) 

499 if self.config.doSkySources: 

500 self.makeSubtask("skySources") 

501 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

502 self.makeSubtask('measurement', schema=self.schema, 

503 algMetadata=self.algMetadata) 

504 if self.config.doNormalizedCalibration: 

505 self.makeSubtask('normalizedCalibrationFlux', schema=self.schema) 

506 self.makeSubtask('postCalibrationMeasurement', schema=self.schema, 

507 algMetadata=self.algMetadata) 

508 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

509 if self.config.doApCorr: 

510 self.makeSubtask('applyApCorr', schema=self.schema) 

511 self.makeSubtask('catalogCalculation', schema=self.schema) 

512 

513 if self.config.doAstrometry: 

514 self.makeSubtask("astrometryDetection", schema=self.schema) 

515 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

516 schema=self.schema) 

517 if self.config.doPhotoCal: 

518 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

519 schema=self.schema) 

520 if self.config.doComputeSummaryStats: 

521 self.makeSubtask('computeSummaryStats') 

522 if self.config.doCreateSummaryMetrics: 

523 self.makeSubtask('createSummaryMetrics') 

524 

525 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

526 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

527 "reference object loaders.") 

528 

529 if self.schemaMapper is not None: 

530 # finalize the schema 

531 self.schema = self.schemaMapper.getOutputSchema() 

532 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

533 

534 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

535 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

536 self.outputSchema = sourceCatSchema 

537 

538 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

539 inputs = butlerQC.get(inputRefs) 

540 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

541 

542 if self.config.doAstrometry: 

543 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

544 for ref in inputRefs.astromRefCat], 

545 refCats=inputs.pop('astromRefCat'), 

546 name=self.config.connections.astromRefCat, 

547 config=self.config.astromRefObjLoader, log=self.log) 

548 self.astrometry.setRefObjLoader(refObjLoader) 

549 

550 if self.config.doPhotoCal: 

551 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

552 for ref in inputRefs.photoRefCat], 

553 refCats=inputs.pop('photoRefCat'), 

554 name=self.config.connections.photoRefCat, 

555 config=self.config.photoRefObjLoader, 

556 log=self.log) 

557 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

558 

559 outputs = self.run(**inputs) 

560 

561 if self.config.doWriteMatches and self.config.doAstrometry: 

562 if outputs.astromMatches is not None: 

563 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

564 normalizedMatches.table.setMetadata(outputs.matchMeta) 

565 if self.config.doWriteMatchesDenormalized: 

566 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

567 outputs.matchesDenormalized = denormMatches 

568 outputs.matches = normalizedMatches 

569 else: 

570 del outputRefs.matches 

571 if self.config.doWriteMatchesDenormalized: 

572 del outputRefs.matchesDenormalized 

573 butlerQC.put(outputs, outputRefs) 

574 

575 @timeMethod 

576 def run(self, exposure, background=None, 

577 icSourceCat=None, idGenerator=None): 

578 """Calibrate an exposure. 

579 

580 Parameters 

581 ---------- 

582 exposure : `lsst.afw.image.ExposureF` 

583 Exposure to calibrate. 

584 background : `lsst.afw.math.BackgroundList`, optional 

585 Initial model of background already subtracted from exposure. 

586 icSourceCat : `lsst.afw.image.SourceCatalog`, optional 

587 SourceCatalog from CharacterizeImageTask from which we can copy 

588 some fields. 

589 idGenerator : `lsst.meas.base.IdGenerator`, optional 

590 Object that generates source IDs and provides RNG seeds. 

591 

592 Returns 

593 ------- 

594 result : `lsst.pipe.base.Struct` 

595 Results as a struct with attributes: 

596 

597 ``exposure`` 

598 Characterized exposure (`lsst.afw.image.ExposureF`). 

599 ``sourceCat`` 

600 Detected sources (`lsst.afw.table.SourceCatalog`). 

601 ``outputBackground`` 

602 Model of subtracted background (`lsst.afw.math.BackgroundList`). 

603 ``astromMatches`` 

604 List of source/ref matches from astrometry solver. 

605 ``matchMeta`` 

606 Metadata from astrometry matches. 

607 ``outputExposure`` 

608 Another reference to ``exposure`` for compatibility. 

609 ``outputCat`` 

610 Another reference to ``sourceCat`` for compatibility. 

611 """ 

612 # detect, deblend and measure sources 

613 if idGenerator is None: 

614 idGenerator = IdGenerator() 

615 

616 if background is None: 

617 background = BackgroundList() 

618 table = SourceTable.make(self.schema, idGenerator.make_table_id_factory()) 

619 table.setMetadata(self.algMetadata) 

620 

621 # perform astrometry calibration: 

622 # fit an improved WCS and update the exposure's WCS in place 

623 astromCat = None 

624 astromMatches = None 

625 matchMeta = None 

626 if self.config.doAstrometry: 

627 try: 

628 # Run a detection specific for deteting astrometry sources 

629 astromDetections = self.astrometryDetection.run( 

630 table=table, exposure=exposure, background=background 

631 ) 

632 astromCat = astromDetections.sources 

633 if not astromCat.isContiguous(): 

634 astromCat = astromCat.copy(deep=True) 

635 self.measurement.run( 

636 measCat=astromCat, 

637 exposure=exposure, 

638 exposureId=idGenerator.catalog_id, 

639 ) 

640 if self.config.doNormalizedCalibration: 

641 self.normalizedCalibrationFlux.run( 

642 exposure=exposure, 

643 catalog=astromCat, 

644 ) 

645 if self.config.doApCorr: 

646 apCorrMap = exposure.getInfo().getApCorrMap() 

647 if apCorrMap is None: 

648 self.log.warning("Image does not have valid aperture correction map for %r; " 

649 "skipping aperture correction", idGenerator.catalog_id) 

650 else: 

651 self.applyApCorr.run( 

652 catalog=astromCat, 

653 apCorrMap=apCorrMap, 

654 ) 

655 self.catalogCalculation.run(astromCat) 

656 

657 self.setPrimaryFlags.run(astromCat) 

658 

659 if icSourceCat is not None and \ 

660 len(self.config.icSourceFieldsToCopy) > 0: 

661 self.copyCalibSourceFields( 

662 calibType="icSource", schemaMapper=self.schemaMapper, calibCat=icSourceCat, 

663 sourceCat=astromCat, fieldsToCopy=self.config.icSourceFieldsToCopy 

664 ) 

665 

666 if not astromCat.isContiguous(): 

667 astromCat = astromCat.copy(deep=True) 

668 

669 astromRes = self.astrometry.run(exposure=exposure, sourceCat=astromCat) 

670 astromMatches = astromRes.matches 

671 matchMeta = astromRes.matchMeta 

672 self.astrometry.check(exposure, astromCat, len(astromMatches)) 

673 

674 except AstrometryError as e: 

675 # Maintain old behavior of not stopping for astrometry errors. 

676 self.log.warning(e) 

677 if exposure.getWcs() is None: 

678 if self.config.requireAstrometry: 

679 raise RuntimeError(f"WCS fit failed for {idGenerator} and requireAstrometry " 

680 "is True.") 

681 else: 

682 self.log.warning("Unable to perform astrometric calibration for %r but " 

683 "requireAstrometry is False: attempting to proceed...", 

684 idGenerator) 

685 

686 detRes = self.detection.run(table=table, exposure=exposure, 

687 doSmooth=True) 

688 

689 self.recordMaskedPixelFractions(exposure) 

690 self.metadata['positive_footprint_count'] = detRes.numPos 

691 self.metadata['negative_footprint_count'] = detRes.numNeg 

692 

693 sourceCat = detRes.sources 

694 # Update the source cooordinates with the current wcs. 

695 if exposure.wcs is not None: 

696 afwTable.updateSourceCoords(exposure.wcs, sourceList=sourceCat) 

697 

698 if detRes.background: 

699 for bg in detRes.background: 

700 background.append(bg) 

701 if self.config.doSkySources: 

702 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id) 

703 if skySourceFootprints: 

704 self.metadata['sky_footprint_count'] = len(skySourceFootprints) 

705 for foot in skySourceFootprints: 

706 s = sourceCat.addNew() 

707 s.setFootprint(foot) 

708 s.set(self.skySourceKey, True) 

709 if self.config.doDeblend: 

710 self.deblend.run(exposure=exposure, sources=sourceCat) 

711 if not sourceCat.isContiguous(): 

712 sourceCat = sourceCat.copy(deep=True) 

713 self.metadata['source_count'] = len(sourceCat) 

714 self.measurement.run( 

715 measCat=sourceCat, 

716 exposure=exposure, 

717 exposureId=idGenerator.catalog_id, 

718 ) 

719 self.metadata['saturated_source_count'] = ( 

720 np.sum(sourceCat['base_PixelFlags_flag_saturated']) 

721 ) 

722 self.metadata['bad_source_count'] = ( 

723 np.sum(sourceCat['base_PixelFlags_flag_bad']) 

724 ) 

725 if self.config.doNormalizedCalibration: 

726 self.normalizedCalibrationFlux.run( 

727 exposure=exposure, 

728 catalog=sourceCat, 

729 ) 

730 if self.config.doApCorr: 

731 apCorrMap = exposure.getInfo().getApCorrMap() 

732 if apCorrMap is None: 

733 self.log.warning("Image does not have valid aperture correction map for %r; " 

734 "skipping aperture correction", idGenerator.catalog_id) 

735 else: 

736 self.applyApCorr.run( 

737 catalog=sourceCat, 

738 apCorrMap=apCorrMap, 

739 ) 

740 self.catalogCalculation.run(sourceCat) 

741 

742 self.setPrimaryFlags.run(sourceCat) 

743 

744 if icSourceCat is not None and \ 

745 len(self.config.icSourceFieldsToCopy) > 0: 

746 self.copyCalibSourceFields(calibType="icSource", schemaMapper=self.schemaMapper, 

747 calibCat=icSourceCat, sourceCat=sourceCat, 

748 fieldsToCopy=self.config.icSourceFieldsToCopy) 

749 

750 if astromCat is not None and \ 

751 len(self.config.astromFieldsToCopy) > 0: 

752 self.copyCalibSourceFields(calibType="astrometry", 

753 schemaMapper=afwTable.SchemaMapper(sourceCat.schema), 

754 calibCat=astromCat, sourceCat=sourceCat, 

755 fieldsToCopy=self.config.astromFieldsToCopy) 

756 

757 # TODO DM-11568: this contiguous check-and-copy could go away if we 

758 # reserve enough space during SourceDetection and/or SourceDeblend. 

759 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

760 # contiguity now, so views are preserved from here on. 

761 if not sourceCat.isContiguous(): 

762 sourceCat = sourceCat.copy(deep=True) 

763 # compute photometric calibration 

764 if self.config.doPhotoCal: 

765 if np.all(np.isnan(sourceCat["coord_ra"])) or np.all(np.isnan(sourceCat["coord_dec"])): 

766 if self.config.requirePhotoCal: 

767 raise RuntimeError(f"Astrometry failed for {idGenerator}, so cannot do " 

768 "photoCal, but requirePhotoCal is True.") 

769 self.log.warning("Astrometry failed for %r, so cannot do photoCal. requirePhotoCal " 

770 "is False, so skipping photometric calibration and setting photoCalib " 

771 "to None. Attempting to proceed...", idGenerator) 

772 exposure.setPhotoCalib(None) 

773 self.setMetadata(exposure=exposure, photoRes=None) 

774 else: 

775 try: 

776 photoRes = self.photoCal.run( 

777 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id 

778 ) 

779 exposure.setPhotoCalib(photoRes.photoCalib) 

780 # TODO: reword this to phrase it in terms of the 

781 # calibration factor? 

782 self.log.info("Photometric zero-point: %f", 

783 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

784 self.setMetadata(exposure=exposure, photoRes=photoRes) 

785 except Exception as e: 

786 if self.config.requirePhotoCal: 

787 raise 

788 self.log.warning("Unable to perform photometric calibration " 

789 "(%s): attempting to proceed", e) 

790 self.setMetadata(exposure=exposure, photoRes=None) 

791 

792 self.postCalibrationMeasurement.run( 

793 measCat=sourceCat, 

794 exposure=exposure, 

795 exposureId=idGenerator.catalog_id, 

796 ) 

797 

798 summaryMetrics = None 

799 if self.config.doComputeSummaryStats: 

800 summary = self.computeSummaryStats.run(exposure=exposure, 

801 sources=sourceCat, 

802 background=background) 

803 exposure.getInfo().setSummaryStats(summary) 

804 if self.config.doCreateSummaryMetrics: 

805 summaryMetrics = self.createSummaryMetrics.run(data=summary.__dict__).metrics 

806 

807 frame = getDebugFrame(self._display, "calibrate") 

808 if frame: 

809 displayAstrometry( 

810 sourceCat=sourceCat, 

811 exposure=exposure, 

812 matches=astromMatches, 

813 frame=frame, 

814 pause=False, 

815 ) 

816 

817 return pipeBase.Struct( 

818 sourceCat=sourceCat, 

819 astromMatches=astromMatches, 

820 matchMeta=matchMeta, 

821 outputExposure=exposure, 

822 outputCat=sourceCat, 

823 outputBackground=background, 

824 outputSummaryMetrics=summaryMetrics 

825 ) 

826 

827 def setMetadata(self, exposure, photoRes=None): 

828 """Set task and exposure metadata. 

829 

830 Logs a warning continues if needed data is missing. 

831 

832 Parameters 

833 ---------- 

834 exposure : `lsst.afw.image.ExposureF` 

835 Exposure to set metadata on. 

836 photoRes : `lsst.pipe.base.Struct`, optional 

837 Result of running photoCal task. 

838 """ 

839 if photoRes is None: 

840 return 

841 

842 metadata = exposure.getMetadata() 

843 

844 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

845 try: 

846 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

847 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

848 except Exception: 

849 self.log.warning("Could not set normalized MAGZERO in header: no " 

850 "exposure time")