Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

41from .fakes import BaseFakeSourcesTask 

42from .photoCal import PhotoCalTask 

43from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask 

44 

45 

46__all__ = ["CalibrateConfig", "CalibrateTask"] 

47 

48 

49class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

50 defaultTemplates={}): 

51 

52 icSourceSchema = cT.InitInput( 

53 doc="Schema produced by characterize image task, used to initialize this task", 

54 name="icSrc_schema", 

55 storageClass="SourceCatalog", 

56 ) 

57 

58 outputSchema = cT.InitOutput( 

59 doc="Schema after CalibrateTask has been initialized", 

60 name="src_schema", 

61 storageClass="SourceCatalog", 

62 ) 

63 

64 exposure = cT.Input( 

65 doc="Input image to calibrate", 

66 name="icExp", 

67 storageClass="ExposureF", 

68 dimensions=("instrument", "visit", "detector"), 

69 ) 

70 

71 background = cT.Input( 

72 doc="Backgrounds determined by characterize task", 

73 name="icExpBackground", 

74 storageClass="Background", 

75 dimensions=("instrument", "visit", "detector"), 

76 ) 

77 

78 icSourceCat = cT.Input( 

79 doc="Source catalog created by characterize task", 

80 name="icSrc", 

81 storageClass="SourceCatalog", 

82 dimensions=("instrument", "visit", "detector"), 

83 ) 

84 

85 astromRefCat = cT.PrerequisiteInput( 

86 doc="Reference catalog to use for astrometry", 

87 name="cal_ref_cat", 

88 storageClass="SimpleCatalog", 

89 dimensions=("skypix",), 

90 deferLoad=True, 

91 multiple=True, 

92 ) 

93 

94 photoRefCat = cT.PrerequisiteInput( 

95 doc="Reference catalog to use for photometric calibration", 

96 name="cal_ref_cat", 

97 storageClass="SimpleCatalog", 

98 dimensions=("skypix",), 

99 deferLoad=True, 

100 multiple=True 

101 ) 

102 

103 outputExposure = cT.Output( 

104 doc="Exposure after running calibration task", 

105 name="calexp", 

106 storageClass="ExposureF", 

107 dimensions=("instrument", "visit", "detector"), 

108 ) 

109 

110 outputCat = cT.Output( 

111 doc="Source catalog produced in calibrate task", 

112 name="src", 

113 storageClass="SourceCatalog", 

114 dimensions=("instrument", "visit", "detector"), 

115 ) 

116 

117 outputBackground = cT.Output( 

118 doc="Background models estimated in calibration task", 

119 name="calexpBackground", 

120 storageClass="Background", 

121 dimensions=("instrument", "visit", "detector"), 

122 ) 

123 

124 matches = cT.Output( 

125 doc="Source/refObj matches from the astrometry solver", 

126 name="srcMatch", 

127 storageClass="Catalog", 

128 dimensions=("instrument", "visit", "detector"), 

129 ) 

130 

131 matchesDenormalized = cT.Output( 

132 doc="Denormalized matches from astrometry solver", 

133 name="srcMatchFull", 

134 storageClass="Catalog", 

135 dimensions=("instrument", "visit", "detector"), 

136 ) 

137 

138 def __init__(self, *, config=None): 

139 super().__init__(config=config) 

140 

141 if config.doAstrometry is False: 

142 self.prerequisiteInputs.remove("astromRefCat") 

143 if config.doPhotoCal is False: 

144 self.prerequisiteInputs.remove("photoRefCat") 

145 

146 if config.doWriteMatches is False or config.doAstrometry is False: 

147 self.outputs.remove("matches") 

148 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

149 self.outputs.remove("matchesDenormalized") 

150 

151 

152class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

153 """Config for CalibrateTask""" 

154 doWrite = pexConfig.Field( 

155 dtype=bool, 

156 default=True, 

157 doc="Save calibration results?", 

158 ) 

159 doWriteHeavyFootprintsInSources = pexConfig.Field( 

160 dtype=bool, 

161 default=True, 

162 doc="Include HeavyFootprint data in source table? If false then heavy " 

163 "footprints are saved as normal footprints, which saves some space" 

164 ) 

165 doWriteMatches = pexConfig.Field( 

166 dtype=bool, 

167 default=True, 

168 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

169 ) 

170 doWriteMatchesDenormalized = pexConfig.Field( 

171 dtype=bool, 

172 default=False, 

173 doc=("Write reference matches in denormalized format? " 

174 "This format uses more disk space, but is more convenient to " 

175 "read. Ignored if doWriteMatches=False or doWrite=False."), 

176 ) 

177 doAstrometry = pexConfig.Field( 

178 dtype=bool, 

179 default=True, 

180 doc="Perform astrometric calibration?", 

181 ) 

182 astromRefObjLoader = pexConfig.ConfigurableField( 

183 target=LoadIndexedReferenceObjectsTask, 

184 doc="reference object loader for astrometric calibration", 

185 ) 

186 photoRefObjLoader = pexConfig.ConfigurableField( 

187 target=LoadIndexedReferenceObjectsTask, 

188 doc="reference object loader for photometric calibration", 

189 ) 

190 astrometry = pexConfig.ConfigurableField( 

191 target=AstrometryTask, 

192 doc="Perform astrometric calibration to refine the WCS", 

193 ) 

194 requireAstrometry = pexConfig.Field( 

195 dtype=bool, 

196 default=True, 

197 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

198 "false."), 

199 ) 

200 doPhotoCal = pexConfig.Field( 

201 dtype=bool, 

202 default=True, 

203 doc="Perform phometric calibration?", 

204 ) 

205 requirePhotoCal = pexConfig.Field( 

206 dtype=bool, 

207 default=True, 

208 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

209 "false."), 

210 ) 

211 photoCal = pexConfig.ConfigurableField( 

212 target=PhotoCalTask, 

213 doc="Perform photometric calibration", 

214 ) 

215 icSourceFieldsToCopy = pexConfig.ListField( 

216 dtype=str, 

217 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

218 doc=("Fields to copy from the icSource catalog to the output catalog " 

219 "for matching sources Any missing fields will trigger a " 

220 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

221 ) 

222 matchRadiusPix = pexConfig.Field( 

223 dtype=float, 

224 default=3, 

225 doc=("Match radius for matching icSourceCat objects to sourceCat " 

226 "objects (pixels)"), 

227 ) 

228 checkUnitsParseStrict = pexConfig.Field( 

229 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

230 "'warn' or 'silent'"), 

231 dtype=str, 

232 default="raise", 

233 ) 

234 detection = pexConfig.ConfigurableField( 

235 target=SourceDetectionTask, 

236 doc="Detect sources" 

237 ) 

238 doDeblend = pexConfig.Field( 

239 dtype=bool, 

240 default=True, 

241 doc="Run deblender input exposure" 

242 ) 

243 deblend = pexConfig.ConfigurableField( 

244 target=SourceDeblendTask, 

245 doc="Split blended sources into their components" 

246 ) 

247 doSkySources = pexConfig.Field( 

248 dtype=bool, 

249 default=True, 

250 doc="Generate sky sources?", 

251 ) 

252 skySources = pexConfig.ConfigurableField( 

253 target=SkyObjectsTask, 

254 doc="Generate sky sources", 

255 ) 

256 measurement = pexConfig.ConfigurableField( 

257 target=SingleFrameMeasurementTask, 

258 doc="Measure sources" 

259 ) 

260 setPrimaryFlags = pexConfig.ConfigurableField( 

261 target=SetPrimaryFlagsTask, 

262 doc=("Set flags for primary source classification in single frame " 

263 "processing. True if sources are not sky sources and not a parent.") 

264 ) 

265 doApCorr = pexConfig.Field( 

266 dtype=bool, 

267 default=True, 

268 doc="Run subtask to apply aperture correction" 

269 ) 

270 applyApCorr = pexConfig.ConfigurableField( 

271 target=ApplyApCorrTask, 

272 doc="Subtask to apply aperture corrections" 

273 ) 

274 # If doApCorr is False, and the exposure does not have apcorrections 

275 # already applied, the active plugins in catalogCalculation almost 

276 # certainly should not contain the characterization plugin 

277 catalogCalculation = pexConfig.ConfigurableField( 

278 target=CatalogCalculationTask, 

279 doc="Subtask to run catalogCalculation plugins on catalog" 

280 ) 

281 doInsertFakes = pexConfig.Field( 

282 dtype=bool, 

283 default=False, 

284 doc="Run fake sources injection task" 

285 ) 

286 insertFakes = pexConfig.ConfigurableField( 

287 target=BaseFakeSourcesTask, 

288 doc="Injection of fake sources for testing purposes (must be " 

289 "retargeted)" 

290 ) 

291 doComputeSummaryStats = pexConfig.Field( 

292 dtype=bool, 

293 default=True, 

294 doc="Run subtask to measure exposure summary statistics?" 

295 ) 

296 computeSummaryStats = pexConfig.ConfigurableField( 

297 target=ComputeExposureSummaryStatsTask, 

298 doc="Subtask to run computeSummaryStats on exposure" 

299 ) 

300 doWriteExposure = pexConfig.Field( 

301 dtype=bool, 

302 default=True, 

303 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

304 "normal calexp but as a fakes_calexp." 

305 ) 

306 

307 def setDefaults(self): 

308 super().setDefaults() 

309 self.detection.doTempLocalBackground = False 

310 self.deblend.maxFootprintSize = 2000 

311 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"] 

312 

313 def validate(self): 

314 super().validate() 

315 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

316 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

317 raise ValueError( 

318 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

319 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

320 ) 

321 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

322 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

323 raise ValueError( 

324 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

325 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

326 ) 

327 

328 

329## \addtogroup LSST_task_documentation 

330## \{ 

331## \page CalibrateTask 

332## \ref CalibrateTask_ "CalibrateTask" 

333## \copybrief CalibrateTask 

334## \} 

335 

336class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

337 r"""!Calibrate an exposure: measure sources and perform astrometric and 

338 photometric calibration 

339 

340 @anchor CalibrateTask_ 

341 

342 @section pipe_tasks_calibrate_Contents Contents 

343 

344 - @ref pipe_tasks_calibrate_Purpose 

345 - @ref pipe_tasks_calibrate_Initialize 

346 - @ref pipe_tasks_calibrate_IO 

347 - @ref pipe_tasks_calibrate_Config 

348 - @ref pipe_tasks_calibrate_Metadata 

349 - @ref pipe_tasks_calibrate_Debug 

350 

351 

352 @section pipe_tasks_calibrate_Purpose Description 

353 

354 Given an exposure with a good PSF model and aperture correction map 

355 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

356 operations: 

357 - Run detection and measurement 

358 - Run astrometry subtask to fit an improved WCS 

359 - Run photoCal subtask to fit the exposure's photometric zero-point 

360 

361 @section pipe_tasks_calibrate_Initialize Task initialisation 

362 

363 @copydoc \_\_init\_\_ 

364 

365 @section pipe_tasks_calibrate_IO Invoking the Task 

366 

367 If you want this task to unpersist inputs or persist outputs, then call 

368 the `runDataRef` method (a wrapper around the `run` method). 

369 

370 If you already have the inputs unpersisted and do not want to persist the 

371 output then it is more direct to call the `run` method: 

372 

373 @section pipe_tasks_calibrate_Config Configuration parameters 

374 

375 See @ref CalibrateConfig 

376 

377 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

378 

379 Exposure metadata 

380 <dl> 

381 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

382 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

383 task 

384 <dt>COLORTERM1 <dd>?? (always 0.0) 

385 <dt>COLORTERM2 <dd>?? (always 0.0) 

386 <dt>COLORTERM3 <dd>?? (always 0.0) 

387 </dl> 

388 

389 @section pipe_tasks_calibrate_Debug Debug variables 

390 

391 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

392 interface supports a flag 

393 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

394 for more about `debug.py`. 

395 

396 CalibrateTask has a debug dictionary containing one key: 

397 <dl> 

398 <dt>calibrate 

399 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

400 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

401 the meaning of the various symbols. 

402 </dl> 

403 

404 For example, put something like: 

405 @code{.py} 

406 import lsstDebug 

407 def DebugInfo(name): 

408 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

409 # call us recursively 

410 if name == "lsst.pipe.tasks.calibrate": 

411 di.display = dict( 

412 calibrate = 1, 

413 ) 

414 

415 return di 

416 

417 lsstDebug.Info = DebugInfo 

418 @endcode 

419 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

420 flag. 

421 

422 Some subtasks may have their own debug variables; see individual Task 

423 documentation. 

424 """ 

425 

426 # Example description used to live here, removed 2-20-2017 as per 

427 # https://jira.lsstcorp.org/browse/DM-9520 

428 

429 ConfigClass = CalibrateConfig 

430 _DefaultName = "calibrate" 

431 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

432 

433 def __init__(self, butler=None, astromRefObjLoader=None, 

434 photoRefObjLoader=None, icSourceSchema=None, 

435 initInputs=None, **kwargs): 

436 """!Construct a CalibrateTask 

437 

438 @param[in] butler The butler is passed to the refObjLoader constructor 

439 in case it is needed. Ignored if the refObjLoader argument 

440 provides a loader directly. 

441 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

442 that supplies an external reference catalog for astrometric 

443 calibration. May be None if the desired loader can be constructed 

444 from the butler argument or all steps requiring a reference catalog 

445 are disabled. 

446 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

447 that supplies an external reference catalog for photometric 

448 calibration. May be None if the desired loader can be constructed 

449 from the butler argument or all steps requiring a reference catalog 

450 are disabled. 

451 @param[in] icSourceSchema schema for icSource catalog, or None. 

452 Schema values specified in config.icSourceFieldsToCopy will be 

453 taken from this schema. If set to None, no values will be 

454 propagated from the icSourceCatalog 

455 @param[in,out] kwargs other keyword arguments for 

456 lsst.pipe.base.CmdLineTask 

457 """ 

458 super().__init__(**kwargs) 

459 

460 if icSourceSchema is None and butler is not None: 

461 # Use butler to read icSourceSchema from disk. 

462 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

463 

464 if icSourceSchema is None and butler is None and initInputs is not None: 

465 icSourceSchema = initInputs['icSourceSchema'].schema 

466 

467 if icSourceSchema is not None: 

468 # use a schema mapper to avoid copying each field separately 

469 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

470 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

471 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

472 

473 # Add fields to copy from an icSource catalog 

474 # and a field to indicate that the source matched a source in that 

475 # catalog. If any fields are missing then raise an exception, but 

476 # first find all missing fields in order to make the error message 

477 # more useful. 

478 self.calibSourceKey = self.schemaMapper.addOutputField( 

479 afwTable.Field["Flag"]("calib_detected", 

480 "Source was detected as an icSource")) 

481 missingFieldNames = [] 

482 for fieldName in self.config.icSourceFieldsToCopy: 

483 try: 

484 schemaItem = icSourceSchema.find(fieldName) 

485 except Exception: 

486 missingFieldNames.append(fieldName) 

487 else: 

488 # field found; if addMapping fails then raise an exception 

489 self.schemaMapper.addMapping(schemaItem.getKey()) 

490 

491 if missingFieldNames: 

492 raise RuntimeError("isSourceCat is missing fields {} " 

493 "specified in icSourceFieldsToCopy" 

494 .format(missingFieldNames)) 

495 

496 # produce a temporary schema to pass to the subtasks; finalize it 

497 # later 

498 self.schema = self.schemaMapper.editOutputSchema() 

499 else: 

500 self.schemaMapper = None 

501 self.schema = afwTable.SourceTable.makeMinimalSchema() 

502 self.makeSubtask('detection', schema=self.schema) 

503 

504 self.algMetadata = dafBase.PropertyList() 

505 

506 # Only create a subtask for fakes if configuration option is set 

507 # N.B. the config for fake object task must be retargeted to a child 

508 # of BaseFakeSourcesTask 

509 if self.config.doInsertFakes: 

510 self.makeSubtask("insertFakes") 

511 

512 if self.config.doDeblend: 

513 self.makeSubtask("deblend", schema=self.schema) 

514 if self.config.doSkySources: 

515 self.makeSubtask("skySources") 

516 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

517 self.makeSubtask('measurement', schema=self.schema, 

518 algMetadata=self.algMetadata) 

519 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

520 if self.config.doApCorr: 

521 self.makeSubtask('applyApCorr', schema=self.schema) 

522 self.makeSubtask('catalogCalculation', schema=self.schema) 

523 

524 if self.config.doAstrometry: 

525 if astromRefObjLoader is None and butler is not None: 

526 self.makeSubtask('astromRefObjLoader', butler=butler) 

527 astromRefObjLoader = self.astromRefObjLoader 

528 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

529 schema=self.schema) 

530 if self.config.doPhotoCal: 

531 if photoRefObjLoader is None and butler is not None: 

532 self.makeSubtask('photoRefObjLoader', butler=butler) 

533 photoRefObjLoader = self.photoRefObjLoader 

534 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

535 schema=self.schema) 

536 if self.config.doComputeSummaryStats: 

537 self.makeSubtask('computeSummaryStats') 

538 

539 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

540 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

541 "reference object loaders.") 

542 

543 if self.schemaMapper is not None: 

544 # finalize the schema 

545 self.schema = self.schemaMapper.getOutputSchema() 

546 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

547 

548 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

549 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

550 self.outputSchema = sourceCatSchema 

551 

552 @pipeBase.timeMethod 

553 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

554 doUnpersist=True): 

555 """!Calibrate an exposure, optionally unpersisting inputs and 

556 persisting outputs. 

557 

558 This is a wrapper around the `run` method that unpersists inputs 

559 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

560 

561 @param[in] dataRef butler data reference corresponding to a science 

562 image 

563 @param[in,out] exposure characterized exposure (an 

564 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

565 icExp and icBackground. See `run` method for details of what is 

566 read and written. 

567 @param[in,out] background initial model of background already 

568 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

569 None if no background has been subtracted, though that is unusual 

570 for calibration. A refined background model is output. Ignored if 

571 exposure is None. 

572 @param[in] icSourceCat catalog from which to copy the fields specified 

573 by icSourceKeys, or None; 

574 @param[in] doUnpersist unpersist data: 

575 - if True, exposure, background and icSourceCat are read from 

576 dataRef and those three arguments must all be None; 

577 - if False the exposure must be provided; background and 

578 icSourceCat are optional. True is intended for running as a 

579 command-line task, False for running as a subtask 

580 @return same data as the calibrate method 

581 """ 

582 self.log.info("Processing %s", dataRef.dataId) 

583 

584 if doUnpersist: 

585 if any(item is not None for item in (exposure, background, 

586 icSourceCat)): 

587 raise RuntimeError("doUnpersist true; exposure, background " 

588 "and icSourceCat must all be None") 

589 exposure = dataRef.get("icExp", immediate=True) 

590 background = dataRef.get("icExpBackground", immediate=True) 

591 icSourceCat = dataRef.get("icSrc", immediate=True) 

592 elif exposure is None: 

593 raise RuntimeError("doUnpersist false; exposure must be provided") 

594 

595 exposureIdInfo = dataRef.get("expIdInfo") 

596 

597 calRes = self.run( 

598 exposure=exposure, 

599 exposureIdInfo=exposureIdInfo, 

600 background=background, 

601 icSourceCat=icSourceCat, 

602 ) 

603 

604 if self.config.doWrite: 

605 self.writeOutputs( 

606 dataRef=dataRef, 

607 exposure=calRes.exposure, 

608 background=calRes.background, 

609 sourceCat=calRes.sourceCat, 

610 astromMatches=calRes.astromMatches, 

611 matchMeta=calRes.matchMeta, 

612 ) 

613 

614 return calRes 

615 

616 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

617 inputs = butlerQC.get(inputRefs) 

618 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

619 

620 if self.config.doAstrometry: 

621 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

622 for ref in inputRefs.astromRefCat], 

623 refCats=inputs.pop('astromRefCat'), 

624 config=self.config.astromRefObjLoader, log=self.log) 

625 self.astrometry.setRefObjLoader(refObjLoader) 

626 

627 if self.config.doPhotoCal: 

628 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

629 for ref in inputRefs.photoRefCat], 

630 refCats=inputs.pop('photoRefCat'), 

631 config=self.config.photoRefObjLoader, 

632 log=self.log) 

633 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

634 

635 outputs = self.run(**inputs) 

636 

637 if self.config.doWriteMatches and self.config.doAstrometry: 

638 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

639 normalizedMatches.table.setMetadata(outputs.matchMeta) 

640 if self.config.doWriteMatchesDenormalized: 

641 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

642 outputs.matchesDenormalized = denormMatches 

643 outputs.matches = normalizedMatches 

644 butlerQC.put(outputs, outputRefs) 

645 

646 @pipeBase.timeMethod 

647 def run(self, exposure, exposureIdInfo=None, background=None, 

648 icSourceCat=None): 

649 """!Calibrate an exposure (science image or coadd) 

650 

651 @param[in,out] exposure exposure to calibrate (an 

652 lsst.afw.image.ExposureF or similar); 

653 in: 

654 - MaskedImage 

655 - Psf 

656 out: 

657 - MaskedImage has background subtracted 

658 - Wcs is replaced 

659 - PhotoCalib is replaced 

660 @param[in] exposureIdInfo ID info for exposure (an 

661 lsst.obs.base.ExposureIdInfo) If not provided, returned 

662 SourceCatalog IDs will not be globally unique. 

663 @param[in,out] background background model already subtracted from 

664 exposure (an lsst.afw.math.BackgroundList). May be None if no 

665 background has been subtracted, though that is unusual for 

666 calibration. A refined background model is output. 

667 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

668 from which we can copy some fields. 

669 

670 @return pipe_base Struct containing these fields: 

671 - exposure calibrate science exposure with refined WCS and PhotoCalib 

672 - background model of background subtracted from exposure (an 

673 lsst.afw.math.BackgroundList) 

674 - sourceCat catalog of measured sources 

675 - astromMatches list of source/refObj matches from the astrometry 

676 solver 

677 """ 

678 # detect, deblend and measure sources 

679 if exposureIdInfo is None: 

680 exposureIdInfo = ExposureIdInfo() 

681 

682 if background is None: 

683 background = BackgroundList() 

684 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

685 table = SourceTable.make(self.schema, sourceIdFactory) 

686 table.setMetadata(self.algMetadata) 

687 

688 detRes = self.detection.run(table=table, exposure=exposure, 

689 doSmooth=True) 

690 sourceCat = detRes.sources 

691 if detRes.fpSets.background: 

692 for bg in detRes.fpSets.background: 

693 background.append(bg) 

694 if self.config.doSkySources: 

695 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

696 if skySourceFootprints: 

697 for foot in skySourceFootprints: 

698 s = sourceCat.addNew() 

699 s.setFootprint(foot) 

700 s.set(self.skySourceKey, True) 

701 if self.config.doDeblend: 

702 self.deblend.run(exposure=exposure, sources=sourceCat) 

703 self.measurement.run( 

704 measCat=sourceCat, 

705 exposure=exposure, 

706 exposureId=exposureIdInfo.expId 

707 ) 

708 if self.config.doApCorr: 

709 self.applyApCorr.run( 

710 catalog=sourceCat, 

711 apCorrMap=exposure.getInfo().getApCorrMap() 

712 ) 

713 self.catalogCalculation.run(sourceCat) 

714 

715 self.setPrimaryFlags.run(sourceCat) 

716 

717 if icSourceCat is not None and \ 

718 len(self.config.icSourceFieldsToCopy) > 0: 

719 self.copyIcSourceFields(icSourceCat=icSourceCat, 

720 sourceCat=sourceCat) 

721 

722 # TODO DM-11568: this contiguous check-and-copy could go away if we 

723 # reserve enough space during SourceDetection and/or SourceDeblend. 

724 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

725 # contiguity now, so views are preserved from here on. 

726 if not sourceCat.isContiguous(): 

727 sourceCat = sourceCat.copy(deep=True) 

728 

729 # perform astrometry calibration: 

730 # fit an improved WCS and update the exposure's WCS in place 

731 astromMatches = None 

732 matchMeta = None 

733 if self.config.doAstrometry: 

734 try: 

735 astromRes = self.astrometry.run( 

736 exposure=exposure, 

737 sourceCat=sourceCat, 

738 ) 

739 astromMatches = astromRes.matches 

740 matchMeta = astromRes.matchMeta 

741 except Exception as e: 

742 if self.config.requireAstrometry: 

743 raise 

744 self.log.warning("Unable to perform astrometric calibration " 

745 "(%s): attempting to proceed", e) 

746 

747 # compute photometric calibration 

748 if self.config.doPhotoCal: 

749 try: 

750 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

751 exposure.setPhotoCalib(photoRes.photoCalib) 

752 # TODO: reword this to phrase it in terms of the calibration factor? 

753 self.log.info("Photometric zero-point: %f", 

754 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

755 self.setMetadata(exposure=exposure, photoRes=photoRes) 

756 except Exception as e: 

757 if self.config.requirePhotoCal: 

758 raise 

759 self.log.warning("Unable to perform photometric calibration " 

760 "(%s): attempting to proceed", e) 

761 self.setMetadata(exposure=exposure, photoRes=None) 

762 

763 if self.config.doInsertFakes: 

764 self.insertFakes.run(exposure, background=background) 

765 

766 table = SourceTable.make(self.schema, sourceIdFactory) 

767 table.setMetadata(self.algMetadata) 

768 

769 detRes = self.detection.run(table=table, exposure=exposure, 

770 doSmooth=True) 

771 sourceCat = detRes.sources 

772 if detRes.fpSets.background: 

773 for bg in detRes.fpSets.background: 

774 background.append(bg) 

775 if self.config.doDeblend: 

776 self.deblend.run(exposure=exposure, sources=sourceCat) 

777 self.measurement.run( 

778 measCat=sourceCat, 

779 exposure=exposure, 

780 exposureId=exposureIdInfo.expId 

781 ) 

782 if self.config.doApCorr: 

783 self.applyApCorr.run( 

784 catalog=sourceCat, 

785 apCorrMap=exposure.getInfo().getApCorrMap() 

786 ) 

787 self.catalogCalculation.run(sourceCat) 

788 

789 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

790 self.copyIcSourceFields(icSourceCat=icSourceCat, 

791 sourceCat=sourceCat) 

792 

793 if self.config.doComputeSummaryStats: 

794 summary = self.computeSummaryStats.run(exposure=exposure, 

795 sources=sourceCat, 

796 background=background) 

797 exposure.getInfo().setSummaryStats(summary) 

798 

799 frame = getDebugFrame(self._display, "calibrate") 

800 if frame: 

801 displayAstrometry( 

802 sourceCat=sourceCat, 

803 exposure=exposure, 

804 matches=astromMatches, 

805 frame=frame, 

806 pause=False, 

807 ) 

808 

809 return pipeBase.Struct( 

810 exposure=exposure, 

811 background=background, 

812 sourceCat=sourceCat, 

813 astromMatches=astromMatches, 

814 matchMeta=matchMeta, 

815 # These are duplicate entries with different names for use with 

816 # gen3 middleware 

817 outputExposure=exposure, 

818 outputCat=sourceCat, 

819 outputBackground=background, 

820 ) 

821 

822 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

823 astromMatches, matchMeta): 

824 """Write output data to the output repository 

825 

826 @param[in] dataRef butler data reference corresponding to a science 

827 image 

828 @param[in] exposure exposure to write 

829 @param[in] background background model for exposure 

830 @param[in] sourceCat catalog of measured sources 

831 @param[in] astromMatches list of source/refObj matches from the 

832 astrometry solver 

833 """ 

834 dataRef.put(sourceCat, "src") 

835 if self.config.doWriteMatches and astromMatches is not None: 

836 normalizedMatches = afwTable.packMatches(astromMatches) 

837 normalizedMatches.table.setMetadata(matchMeta) 

838 dataRef.put(normalizedMatches, "srcMatch") 

839 if self.config.doWriteMatchesDenormalized: 

840 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

841 dataRef.put(denormMatches, "srcMatchFull") 

842 if self.config.doWriteExposure: 

843 dataRef.put(exposure, "calexp") 

844 dataRef.put(background, "calexpBackground") 

845 

846 def getSchemaCatalogs(self): 

847 """Return a dict of empty catalogs for each catalog dataset produced 

848 by this task. 

849 """ 

850 sourceCat = afwTable.SourceCatalog(self.schema) 

851 sourceCat.getTable().setMetadata(self.algMetadata) 

852 return {"src": sourceCat} 

853 

854 def setMetadata(self, exposure, photoRes=None): 

855 """!Set task and exposure metadata 

856 

857 Logs a warning and continues if needed data is missing. 

858 

859 @param[in,out] exposure exposure whose metadata is to be set 

860 @param[in] photoRes results of running photoCal; if None then it was 

861 not run 

862 """ 

863 if photoRes is None: 

864 return 

865 

866 metadata = exposure.getMetadata() 

867 

868 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

869 try: 

870 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

871 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

872 except Exception: 

873 self.log.warning("Could not set normalized MAGZERO in header: no " 

874 "exposure time") 

875 magZero = math.nan 

876 

877 try: 

878 metadata.set('MAGZERO', magZero) 

879 metadata.set('MAGZERO_RMS', photoRes.sigma) 

880 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

881 metadata.set('COLORTERM1', 0.0) 

882 metadata.set('COLORTERM2', 0.0) 

883 metadata.set('COLORTERM3', 0.0) 

884 except Exception as e: 

885 self.log.warning("Could not set exposure metadata: %s", e) 

886 

887 def copyIcSourceFields(self, icSourceCat, sourceCat): 

888 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

889 

890 @param[in] icSourceCat catalog from which to copy fields 

891 @param[in,out] sourceCat catalog to which to copy fields 

892 

893 The fields copied are those specified by `config.icSourceFieldsToCopy` 

894 that actually exist in the schema. This was set up by the constructor 

895 using self.schemaMapper. 

896 """ 

897 if self.schemaMapper is None: 

898 raise RuntimeError("To copy icSource fields you must specify " 

899 "icSourceSchema nd icSourceKeys when " 

900 "constructing this task") 

901 if icSourceCat is None or sourceCat is None: 

902 raise RuntimeError("icSourceCat and sourceCat must both be " 

903 "specified") 

904 if len(self.config.icSourceFieldsToCopy) == 0: 

905 self.log.warning("copyIcSourceFields doing nothing because " 

906 "icSourceFieldsToCopy is empty") 

907 return 

908 

909 mc = afwTable.MatchControl() 

910 mc.findOnlyClosest = False # return all matched objects 

911 matches = afwTable.matchXy(icSourceCat, sourceCat, 

912 self.config.matchRadiusPix, mc) 

913 if self.config.doDeblend: 

914 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

915 # if deblended, keep children 

916 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

917 

918 # Because we had to allow multiple matches to handle parents, we now 

919 # need to prune to the best matches 

920 # closest matches as a dict of icSourceCat source ID: 

921 # (icSourceCat source, sourceCat source, distance in pixels) 

922 bestMatches = {} 

923 for m0, m1, d in matches: 

924 id0 = m0.getId() 

925 match = bestMatches.get(id0) 

926 if match is None or d <= match[2]: 

927 bestMatches[id0] = (m0, m1, d) 

928 matches = list(bestMatches.values()) 

929 

930 # Check that no sourceCat sources are listed twice (we already know 

931 # that each match has a unique icSourceCat source ID, due to using 

932 # that ID as the key in bestMatches) 

933 numMatches = len(matches) 

934 numUniqueSources = len(set(m[1].getId() for m in matches)) 

935 if numUniqueSources != numMatches: 

936 self.log.warning("%d icSourceCat sources matched only %d sourceCat " 

937 "sources", numMatches, numUniqueSources) 

938 

939 self.log.info("Copying flags from icSourceCat to sourceCat for " 

940 "%d sources", numMatches) 

941 

942 # For each match: set the calibSourceKey flag and copy the desired 

943 # fields 

944 for icSrc, src, d in matches: 

945 src.setFlag(self.calibSourceKey, True) 

946 # src.assign copies the footprint from icSrc, which we don't want 

947 # (DM-407) 

948 # so set icSrc's footprint to src's footprint before src.assign, 

949 # then restore it 

950 icSrcFootprint = icSrc.getFootprint() 

951 try: 

952 icSrc.setFootprint(src.getFootprint()) 

953 src.assign(icSrc, self.schemaMapper) 

954 finally: 

955 icSrc.setFootprint(icSrcFootprint)