Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask 

41from .fakes import BaseFakeSourcesTask 

42from .photoCal import PhotoCalTask 

43 

44__all__ = ["CalibrateConfig", "CalibrateTask"] 

45 

46 

47class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

48 defaultTemplates={}): 

49 

50 icSourceSchema = cT.InitInput( 

51 doc="Schema produced by characterize image task, used to initialize this task", 

52 name="icSrc_schema", 

53 storageClass="SourceCatalog", 

54 ) 

55 

56 outputSchema = cT.InitOutput( 

57 doc="Schema after CalibrateTask has been initialized", 

58 name="src_schema", 

59 storageClass="SourceCatalog", 

60 ) 

61 

62 exposure = cT.Input( 

63 doc="Input image to calibrate", 

64 name="icExp", 

65 storageClass="ExposureF", 

66 dimensions=("instrument", "visit", "detector"), 

67 ) 

68 

69 background = cT.Input( 

70 doc="Backgrounds determined by characterize task", 

71 name="icExpBackground", 

72 storageClass="Background", 

73 dimensions=("instrument", "visit", "detector"), 

74 ) 

75 

76 icSourceCat = cT.Input( 

77 doc="Source catalog created by characterize task", 

78 name="icSrc", 

79 storageClass="SourceCatalog", 

80 dimensions=("instrument", "visit", "detector"), 

81 ) 

82 

83 astromRefCat = cT.PrerequisiteInput( 

84 doc="Reference catalog to use for astrometry", 

85 name="cal_ref_cat", 

86 storageClass="SimpleCatalog", 

87 dimensions=("skypix",), 

88 deferLoad=True, 

89 multiple=True, 

90 ) 

91 

92 photoRefCat = cT.PrerequisiteInput( 

93 doc="Reference catalog to use for photometric calibration", 

94 name="cal_ref_cat", 

95 storageClass="SimpleCatalog", 

96 dimensions=("skypix",), 

97 deferLoad=True, 

98 multiple=True 

99 ) 

100 

101 outputExposure = cT.Output( 

102 doc="Exposure after running calibration task", 

103 name="calexp", 

104 storageClass="ExposureF", 

105 dimensions=("instrument", "visit", "detector"), 

106 ) 

107 

108 outputCat = cT.Output( 

109 doc="Source catalog produced in calibrate task", 

110 name="src", 

111 storageClass="SourceCatalog", 

112 dimensions=("instrument", "visit", "detector"), 

113 ) 

114 

115 outputBackground = cT.Output( 

116 doc="Background models estimated in calibration task", 

117 name="calexpBackground", 

118 storageClass="Background", 

119 dimensions=("instrument", "visit", "detector"), 

120 ) 

121 

122 matches = cT.Output( 

123 doc="Source/refObj matches from the astrometry solver", 

124 name="srcMatch", 

125 storageClass="Catalog", 

126 dimensions=("instrument", "visit", "detector"), 

127 ) 

128 

129 matchesDenormalized = cT.Output( 

130 doc="Denormalized matches from astrometry solver", 

131 name="srcMatchFull", 

132 storageClass="Catalog", 

133 dimensions=("instrument", "visit", "detector"), 

134 ) 

135 

136 def __init__(self, *, config=None): 

137 super().__init__(config=config) 

138 

139 if config.doAstrometry is False: 

140 self.prerequisiteInputs.remove("astromRefCat") 

141 if config.doPhotoCal is False: 

142 self.prerequisiteInputs.remove("photoRefCat") 

143 

144 if config.doWriteMatches is False or config.doAstrometry is False: 

145 self.outputs.remove("matches") 

146 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

147 self.outputs.remove("matchesDenormalized") 

148 

149 

150class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

151 """Config for CalibrateTask""" 

152 doWrite = pexConfig.Field( 

153 dtype=bool, 

154 default=True, 

155 doc="Save calibration results?", 

156 ) 

157 doWriteHeavyFootprintsInSources = pexConfig.Field( 

158 dtype=bool, 

159 default=True, 

160 doc="Include HeavyFootprint data in source table? If false then heavy " 

161 "footprints are saved as normal footprints, which saves some space" 

162 ) 

163 doWriteMatches = pexConfig.Field( 

164 dtype=bool, 

165 default=True, 

166 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

167 ) 

168 doWriteMatchesDenormalized = pexConfig.Field( 

169 dtype=bool, 

170 default=False, 

171 doc=("Write reference matches in denormalized format? " 

172 "This format uses more disk space, but is more convenient to " 

173 "read. Ignored if doWriteMatches=False or doWrite=False."), 

174 ) 

175 doAstrometry = pexConfig.Field( 

176 dtype=bool, 

177 default=True, 

178 doc="Perform astrometric calibration?", 

179 ) 

180 astromRefObjLoader = pexConfig.ConfigurableField( 

181 target=LoadIndexedReferenceObjectsTask, 

182 doc="reference object loader for astrometric calibration", 

183 ) 

184 photoRefObjLoader = pexConfig.ConfigurableField( 

185 target=LoadIndexedReferenceObjectsTask, 

186 doc="reference object loader for photometric calibration", 

187 ) 

188 astrometry = pexConfig.ConfigurableField( 

189 target=AstrometryTask, 

190 doc="Perform astrometric calibration to refine the WCS", 

191 ) 

192 requireAstrometry = pexConfig.Field( 

193 dtype=bool, 

194 default=True, 

195 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

196 "false."), 

197 ) 

198 doPhotoCal = pexConfig.Field( 

199 dtype=bool, 

200 default=True, 

201 doc="Perform phometric calibration?", 

202 ) 

203 requirePhotoCal = pexConfig.Field( 

204 dtype=bool, 

205 default=True, 

206 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

207 "false."), 

208 ) 

209 photoCal = pexConfig.ConfigurableField( 

210 target=PhotoCalTask, 

211 doc="Perform photometric calibration", 

212 ) 

213 icSourceFieldsToCopy = pexConfig.ListField( 

214 dtype=str, 

215 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

216 doc=("Fields to copy from the icSource catalog to the output catalog " 

217 "for matching sources Any missing fields will trigger a " 

218 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

219 ) 

220 matchRadiusPix = pexConfig.Field( 

221 dtype=float, 

222 default=3, 

223 doc=("Match radius for matching icSourceCat objects to sourceCat " 

224 "objects (pixels)"), 

225 ) 

226 checkUnitsParseStrict = pexConfig.Field( 

227 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

228 "'warn' or 'silent'"), 

229 dtype=str, 

230 default="raise", 

231 ) 

232 detection = pexConfig.ConfigurableField( 

233 target=SourceDetectionTask, 

234 doc="Detect sources" 

235 ) 

236 doDeblend = pexConfig.Field( 

237 dtype=bool, 

238 default=True, 

239 doc="Run deblender input exposure" 

240 ) 

241 deblend = pexConfig.ConfigurableField( 

242 target=SourceDeblendTask, 

243 doc="Split blended sources into their components" 

244 ) 

245 doSkySources = pexConfig.Field( 

246 dtype=bool, 

247 default=True, 

248 doc="Generate sky sources?", 

249 ) 

250 skySources = pexConfig.ConfigurableField( 

251 target=SkyObjectsTask, 

252 doc="Generate sky sources", 

253 ) 

254 measurement = pexConfig.ConfigurableField( 

255 target=SingleFrameMeasurementTask, 

256 doc="Measure sources" 

257 ) 

258 setPrimaryFlags = pexConfig.ConfigurableField( 

259 target=SetPrimaryFlagsTask, 

260 doc=("Set flags for primary source classification in single frame " 

261 "processing. True if sources are not sky sources and not a parent.") 

262 ) 

263 doApCorr = pexConfig.Field( 

264 dtype=bool, 

265 default=True, 

266 doc="Run subtask to apply aperture correction" 

267 ) 

268 applyApCorr = pexConfig.ConfigurableField( 

269 target=ApplyApCorrTask, 

270 doc="Subtask to apply aperture corrections" 

271 ) 

272 # If doApCorr is False, and the exposure does not have apcorrections 

273 # already applied, the active plugins in catalogCalculation almost 

274 # certainly should not contain the characterization plugin 

275 catalogCalculation = pexConfig.ConfigurableField( 

276 target=CatalogCalculationTask, 

277 doc="Subtask to run catalogCalculation plugins on catalog" 

278 ) 

279 doInsertFakes = pexConfig.Field( 

280 dtype=bool, 

281 default=False, 

282 doc="Run fake sources injection task" 

283 ) 

284 insertFakes = pexConfig.ConfigurableField( 

285 target=BaseFakeSourcesTask, 

286 doc="Injection of fake sources for testing purposes (must be " 

287 "retargeted)" 

288 ) 

289 doWriteExposure = pexConfig.Field( 

290 dtype=bool, 

291 default=True, 

292 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

293 "normal calexp but as a fakes_calexp." 

294 ) 

295 

296 def setDefaults(self): 

297 super().setDefaults() 

298 self.detection.doTempLocalBackground = False 

299 self.deblend.maxFootprintSize = 2000 

300 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"] 

301 

302 def validate(self): 

303 super().validate() 

304 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

305 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

306 raise ValueError( 

307 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

308 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

309 ) 

310 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

311 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

312 raise ValueError( 

313 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

314 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

315 ) 

316 

317 

318## \addtogroup LSST_task_documentation 

319## \{ 

320## \page CalibrateTask 

321## \ref CalibrateTask_ "CalibrateTask" 

322## \copybrief CalibrateTask 

323## \} 

324 

325class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

326 r"""!Calibrate an exposure: measure sources and perform astrometric and 

327 photometric calibration 

328 

329 @anchor CalibrateTask_ 

330 

331 @section pipe_tasks_calibrate_Contents Contents 

332 

333 - @ref pipe_tasks_calibrate_Purpose 

334 - @ref pipe_tasks_calibrate_Initialize 

335 - @ref pipe_tasks_calibrate_IO 

336 - @ref pipe_tasks_calibrate_Config 

337 - @ref pipe_tasks_calibrate_Metadata 

338 - @ref pipe_tasks_calibrate_Debug 

339 

340 

341 @section pipe_tasks_calibrate_Purpose Description 

342 

343 Given an exposure with a good PSF model and aperture correction map 

344 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

345 operations: 

346 - Run detection and measurement 

347 - Run astrometry subtask to fit an improved WCS 

348 - Run photoCal subtask to fit the exposure's photometric zero-point 

349 

350 @section pipe_tasks_calibrate_Initialize Task initialisation 

351 

352 @copydoc \_\_init\_\_ 

353 

354 @section pipe_tasks_calibrate_IO Invoking the Task 

355 

356 If you want this task to unpersist inputs or persist outputs, then call 

357 the `runDataRef` method (a wrapper around the `run` method). 

358 

359 If you already have the inputs unpersisted and do not want to persist the 

360 output then it is more direct to call the `run` method: 

361 

362 @section pipe_tasks_calibrate_Config Configuration parameters 

363 

364 See @ref CalibrateConfig 

365 

366 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

367 

368 Exposure metadata 

369 <dl> 

370 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

371 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

372 task 

373 <dt>COLORTERM1 <dd>?? (always 0.0) 

374 <dt>COLORTERM2 <dd>?? (always 0.0) 

375 <dt>COLORTERM3 <dd>?? (always 0.0) 

376 </dl> 

377 

378 @section pipe_tasks_calibrate_Debug Debug variables 

379 

380 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

381 interface supports a flag 

382 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

383 for more about `debug.py`. 

384 

385 CalibrateTask has a debug dictionary containing one key: 

386 <dl> 

387 <dt>calibrate 

388 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

389 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

390 the meaning of the various symbols. 

391 </dl> 

392 

393 For example, put something like: 

394 @code{.py} 

395 import lsstDebug 

396 def DebugInfo(name): 

397 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

398 # call us recursively 

399 if name == "lsst.pipe.tasks.calibrate": 

400 di.display = dict( 

401 calibrate = 1, 

402 ) 

403 

404 return di 

405 

406 lsstDebug.Info = DebugInfo 

407 @endcode 

408 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

409 flag. 

410 

411 Some subtasks may have their own debug variables; see individual Task 

412 documentation. 

413 """ 

414 

415 # Example description used to live here, removed 2-20-2017 as per 

416 # https://jira.lsstcorp.org/browse/DM-9520 

417 

418 ConfigClass = CalibrateConfig 

419 _DefaultName = "calibrate" 

420 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

421 

422 def __init__(self, butler=None, astromRefObjLoader=None, 

423 photoRefObjLoader=None, icSourceSchema=None, 

424 initInputs=None, **kwargs): 

425 """!Construct a CalibrateTask 

426 

427 @param[in] butler The butler is passed to the refObjLoader constructor 

428 in case it is needed. Ignored if the refObjLoader argument 

429 provides a loader directly. 

430 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

431 that supplies an external reference catalog for astrometric 

432 calibration. May be None if the desired loader can be constructed 

433 from the butler argument or all steps requiring a reference catalog 

434 are disabled. 

435 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

436 that supplies an external reference catalog for photometric 

437 calibration. May be None if the desired loader can be constructed 

438 from the butler argument or all steps requiring a reference catalog 

439 are disabled. 

440 @param[in] icSourceSchema schema for icSource catalog, or None. 

441 Schema values specified in config.icSourceFieldsToCopy will be 

442 taken from this schema. If set to None, no values will be 

443 propagated from the icSourceCatalog 

444 @param[in,out] kwargs other keyword arguments for 

445 lsst.pipe.base.CmdLineTask 

446 """ 

447 super().__init__(**kwargs) 

448 

449 if icSourceSchema is None and butler is not None: 

450 # Use butler to read icSourceSchema from disk. 

451 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

452 

453 if icSourceSchema is None and butler is None and initInputs is not None: 

454 icSourceSchema = initInputs['icSourceSchema'].schema 

455 

456 if icSourceSchema is not None: 

457 # use a schema mapper to avoid copying each field separately 

458 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

459 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

460 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

461 

462 # Add fields to copy from an icSource catalog 

463 # and a field to indicate that the source matched a source in that 

464 # catalog. If any fields are missing then raise an exception, but 

465 # first find all missing fields in order to make the error message 

466 # more useful. 

467 self.calibSourceKey = self.schemaMapper.addOutputField( 

468 afwTable.Field["Flag"]("calib_detected", 

469 "Source was detected as an icSource")) 

470 missingFieldNames = [] 

471 for fieldName in self.config.icSourceFieldsToCopy: 

472 try: 

473 schemaItem = icSourceSchema.find(fieldName) 

474 except Exception: 

475 missingFieldNames.append(fieldName) 

476 else: 

477 # field found; if addMapping fails then raise an exception 

478 self.schemaMapper.addMapping(schemaItem.getKey()) 

479 

480 if missingFieldNames: 

481 raise RuntimeError("isSourceCat is missing fields {} " 

482 "specified in icSourceFieldsToCopy" 

483 .format(missingFieldNames)) 

484 

485 # produce a temporary schema to pass to the subtasks; finalize it 

486 # later 

487 self.schema = self.schemaMapper.editOutputSchema() 

488 else: 

489 self.schemaMapper = None 

490 self.schema = afwTable.SourceTable.makeMinimalSchema() 

491 self.makeSubtask('detection', schema=self.schema) 

492 

493 self.algMetadata = dafBase.PropertyList() 

494 

495 # Only create a subtask for fakes if configuration option is set 

496 # N.B. the config for fake object task must be retargeted to a child 

497 # of BaseFakeSourcesTask 

498 if self.config.doInsertFakes: 

499 self.makeSubtask("insertFakes") 

500 

501 if self.config.doDeblend: 

502 self.makeSubtask("deblend", schema=self.schema) 

503 if self.config.doSkySources: 

504 self.makeSubtask("skySources") 

505 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

506 self.makeSubtask('measurement', schema=self.schema, 

507 algMetadata=self.algMetadata) 

508 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True) 

509 if self.config.doApCorr: 

510 self.makeSubtask('applyApCorr', schema=self.schema) 

511 self.makeSubtask('catalogCalculation', schema=self.schema) 

512 

513 if self.config.doAstrometry: 

514 if astromRefObjLoader is None and butler is not None: 

515 self.makeSubtask('astromRefObjLoader', butler=butler) 

516 astromRefObjLoader = self.astromRefObjLoader 

517 self.pixelMargin = astromRefObjLoader.config.pixelMargin 

518 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

519 schema=self.schema) 

520 if self.config.doPhotoCal: 

521 if photoRefObjLoader is None and butler is not None: 

522 self.makeSubtask('photoRefObjLoader', butler=butler) 

523 photoRefObjLoader = self.photoRefObjLoader 

524 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

525 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

526 schema=self.schema) 

527 

528 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

529 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

530 "reference object loaders.") 

531 

532 if self.schemaMapper is not None: 

533 # finalize the schema 

534 self.schema = self.schemaMapper.getOutputSchema() 

535 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

536 

537 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

538 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

539 self.outputSchema = sourceCatSchema 

540 

541 @pipeBase.timeMethod 

542 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

543 doUnpersist=True): 

544 """!Calibrate an exposure, optionally unpersisting inputs and 

545 persisting outputs. 

546 

547 This is a wrapper around the `run` method that unpersists inputs 

548 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

549 

550 @param[in] dataRef butler data reference corresponding to a science 

551 image 

552 @param[in,out] exposure characterized exposure (an 

553 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

554 icExp and icBackground. See `run` method for details of what is 

555 read and written. 

556 @param[in,out] background initial model of background already 

557 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

558 None if no background has been subtracted, though that is unusual 

559 for calibration. A refined background model is output. Ignored if 

560 exposure is None. 

561 @param[in] icSourceCat catalog from which to copy the fields specified 

562 by icSourceKeys, or None; 

563 @param[in] doUnpersist unpersist data: 

564 - if True, exposure, background and icSourceCat are read from 

565 dataRef and those three arguments must all be None; 

566 - if False the exposure must be provided; background and 

567 icSourceCat are optional. True is intended for running as a 

568 command-line task, False for running as a subtask 

569 @return same data as the calibrate method 

570 """ 

571 self.log.info("Processing %s" % (dataRef.dataId)) 

572 

573 if doUnpersist: 

574 if any(item is not None for item in (exposure, background, 

575 icSourceCat)): 

576 raise RuntimeError("doUnpersist true; exposure, background " 

577 "and icSourceCat must all be None") 

578 exposure = dataRef.get("icExp", immediate=True) 

579 background = dataRef.get("icExpBackground", immediate=True) 

580 icSourceCat = dataRef.get("icSrc", immediate=True) 

581 elif exposure is None: 

582 raise RuntimeError("doUnpersist false; exposure must be provided") 

583 

584 exposureIdInfo = dataRef.get("expIdInfo") 

585 

586 calRes = self.run( 

587 exposure=exposure, 

588 exposureIdInfo=exposureIdInfo, 

589 background=background, 

590 icSourceCat=icSourceCat, 

591 ) 

592 

593 if self.config.doWrite: 

594 self.writeOutputs( 

595 dataRef=dataRef, 

596 exposure=calRes.exposure, 

597 background=calRes.background, 

598 sourceCat=calRes.sourceCat, 

599 astromMatches=calRes.astromMatches, 

600 matchMeta=calRes.matchMeta, 

601 ) 

602 

603 return calRes 

604 

605 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

606 inputs = butlerQC.get(inputRefs) 

607 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector") 

608 

609 if self.config.doAstrometry: 

610 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

611 for ref in inputRefs.astromRefCat], 

612 refCats=inputs.pop('astromRefCat'), 

613 config=self.config.astromRefObjLoader, log=self.log) 

614 self.pixelMargin = refObjLoader.config.pixelMargin 

615 self.astrometry.setRefObjLoader(refObjLoader) 

616 

617 if self.config.doPhotoCal: 

618 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

619 for ref in inputRefs.photoRefCat], 

620 refCats=inputs.pop('photoRefCat'), 

621 config=self.config.photoRefObjLoader, 

622 log=self.log) 

623 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

624 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

625 

626 outputs = self.run(**inputs) 

627 

628 if self.config.doWriteMatches and self.config.doAstrometry: 

629 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

630 normalizedMatches.table.setMetadata(outputs.matchMeta) 

631 if self.config.doWriteMatchesDenormalized: 

632 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

633 outputs.matchesDenormalized = denormMatches 

634 outputs.matches = normalizedMatches 

635 butlerQC.put(outputs, outputRefs) 

636 

637 @pipeBase.timeMethod 

638 def run(self, exposure, exposureIdInfo=None, background=None, 

639 icSourceCat=None): 

640 """!Calibrate an exposure (science image or coadd) 

641 

642 @param[in,out] exposure exposure to calibrate (an 

643 lsst.afw.image.ExposureF or similar); 

644 in: 

645 - MaskedImage 

646 - Psf 

647 out: 

648 - MaskedImage has background subtracted 

649 - Wcs is replaced 

650 - PhotoCalib is replaced 

651 @param[in] exposureIdInfo ID info for exposure (an 

652 lsst.obs.base.ExposureIdInfo) If not provided, returned 

653 SourceCatalog IDs will not be globally unique. 

654 @param[in,out] background background model already subtracted from 

655 exposure (an lsst.afw.math.BackgroundList). May be None if no 

656 background has been subtracted, though that is unusual for 

657 calibration. A refined background model is output. 

658 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

659 from which we can copy some fields. 

660 

661 @return pipe_base Struct containing these fields: 

662 - exposure calibrate science exposure with refined WCS and PhotoCalib 

663 - background model of background subtracted from exposure (an 

664 lsst.afw.math.BackgroundList) 

665 - sourceCat catalog of measured sources 

666 - astromMatches list of source/refObj matches from the astrometry 

667 solver 

668 """ 

669 # detect, deblend and measure sources 

670 if exposureIdInfo is None: 

671 exposureIdInfo = ExposureIdInfo() 

672 

673 if background is None: 

674 background = BackgroundList() 

675 sourceIdFactory = exposureIdInfo.makeSourceIdFactory() 

676 table = SourceTable.make(self.schema, sourceIdFactory) 

677 table.setMetadata(self.algMetadata) 

678 

679 detRes = self.detection.run(table=table, exposure=exposure, 

680 doSmooth=True) 

681 sourceCat = detRes.sources 

682 if detRes.fpSets.background: 

683 for bg in detRes.fpSets.background: 

684 background.append(bg) 

685 if self.config.doSkySources: 

686 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

687 if skySourceFootprints: 

688 for foot in skySourceFootprints: 

689 s = sourceCat.addNew() 

690 s.setFootprint(foot) 

691 s.set(self.skySourceKey, True) 

692 if self.config.doDeblend: 

693 self.deblend.run(exposure=exposure, sources=sourceCat) 

694 self.measurement.run( 

695 measCat=sourceCat, 

696 exposure=exposure, 

697 exposureId=exposureIdInfo.expId 

698 ) 

699 if self.config.doApCorr: 

700 self.applyApCorr.run( 

701 catalog=sourceCat, 

702 apCorrMap=exposure.getInfo().getApCorrMap() 

703 ) 

704 self.catalogCalculation.run(sourceCat) 

705 

706 self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend) 

707 

708 if icSourceCat is not None and \ 

709 len(self.config.icSourceFieldsToCopy) > 0: 

710 self.copyIcSourceFields(icSourceCat=icSourceCat, 

711 sourceCat=sourceCat) 

712 

713 # TODO DM-11568: this contiguous check-and-copy could go away if we 

714 # reserve enough space during SourceDetection and/or SourceDeblend. 

715 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

716 # contiguity now, so views are preserved from here on. 

717 if not sourceCat.isContiguous(): 

718 sourceCat = sourceCat.copy(deep=True) 

719 

720 # perform astrometry calibration: 

721 # fit an improved WCS and update the exposure's WCS in place 

722 astromMatches = None 

723 matchMeta = None 

724 if self.config.doAstrometry: 

725 try: 

726 astromRes = self.astrometry.run( 

727 exposure=exposure, 

728 sourceCat=sourceCat, 

729 ) 

730 astromMatches = astromRes.matches 

731 matchMeta = astromRes.matchMeta 

732 except Exception as e: 

733 if self.config.requireAstrometry: 

734 raise 

735 self.log.warn("Unable to perform astrometric calibration " 

736 "(%s): attempting to proceed" % e) 

737 

738 # compute photometric calibration 

739 if self.config.doPhotoCal: 

740 try: 

741 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

742 exposure.setPhotoCalib(photoRes.photoCalib) 

743 # TODO: reword this to phrase it in terms of the calibration factor? 

744 self.log.info("Photometric zero-point: %f" % 

745 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

746 self.setMetadata(exposure=exposure, photoRes=photoRes) 

747 except Exception as e: 

748 if self.config.requirePhotoCal: 

749 raise 

750 self.log.warn("Unable to perform photometric calibration " 

751 "(%s): attempting to proceed" % e) 

752 self.setMetadata(exposure=exposure, photoRes=None) 

753 

754 if self.config.doInsertFakes: 

755 self.insertFakes.run(exposure, background=background) 

756 

757 table = SourceTable.make(self.schema, sourceIdFactory) 

758 table.setMetadata(self.algMetadata) 

759 

760 detRes = self.detection.run(table=table, exposure=exposure, 

761 doSmooth=True) 

762 sourceCat = detRes.sources 

763 if detRes.fpSets.background: 

764 for bg in detRes.fpSets.background: 

765 background.append(bg) 

766 if self.config.doDeblend: 

767 self.deblend.run(exposure=exposure, sources=sourceCat) 

768 self.measurement.run( 

769 measCat=sourceCat, 

770 exposure=exposure, 

771 exposureId=exposureIdInfo.expId 

772 ) 

773 if self.config.doApCorr: 

774 self.applyApCorr.run( 

775 catalog=sourceCat, 

776 apCorrMap=exposure.getInfo().getApCorrMap() 

777 ) 

778 self.catalogCalculation.run(sourceCat) 

779 

780 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

781 self.copyIcSourceFields(icSourceCat=icSourceCat, 

782 sourceCat=sourceCat) 

783 

784 frame = getDebugFrame(self._display, "calibrate") 

785 if frame: 

786 displayAstrometry( 

787 sourceCat=sourceCat, 

788 exposure=exposure, 

789 matches=astromMatches, 

790 frame=frame, 

791 pause=False, 

792 ) 

793 

794 return pipeBase.Struct( 

795 exposure=exposure, 

796 background=background, 

797 sourceCat=sourceCat, 

798 astromMatches=astromMatches, 

799 matchMeta=matchMeta, 

800 # These are duplicate entries with different names for use with 

801 # gen3 middleware 

802 outputExposure=exposure, 

803 outputCat=sourceCat, 

804 outputBackground=background, 

805 ) 

806 

807 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

808 astromMatches, matchMeta): 

809 """Write output data to the output repository 

810 

811 @param[in] dataRef butler data reference corresponding to a science 

812 image 

813 @param[in] exposure exposure to write 

814 @param[in] background background model for exposure 

815 @param[in] sourceCat catalog of measured sources 

816 @param[in] astromMatches list of source/refObj matches from the 

817 astrometry solver 

818 """ 

819 dataRef.put(sourceCat, "src") 

820 if self.config.doWriteMatches and astromMatches is not None: 

821 normalizedMatches = afwTable.packMatches(astromMatches) 

822 normalizedMatches.table.setMetadata(matchMeta) 

823 dataRef.put(normalizedMatches, "srcMatch") 

824 if self.config.doWriteMatchesDenormalized: 

825 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

826 dataRef.put(denormMatches, "srcMatchFull") 

827 if self.config.doWriteExposure: 

828 dataRef.put(exposure, "calexp") 

829 dataRef.put(background, "calexpBackground") 

830 

831 def getSchemaCatalogs(self): 

832 """Return a dict of empty catalogs for each catalog dataset produced 

833 by this task. 

834 """ 

835 sourceCat = afwTable.SourceCatalog(self.schema) 

836 sourceCat.getTable().setMetadata(self.algMetadata) 

837 return {"src": sourceCat} 

838 

839 def setMetadata(self, exposure, photoRes=None): 

840 """!Set task and exposure metadata 

841 

842 Logs a warning and continues if needed data is missing. 

843 

844 @param[in,out] exposure exposure whose metadata is to be set 

845 @param[in] photoRes results of running photoCal; if None then it was 

846 not run 

847 """ 

848 if photoRes is None: 

849 return 

850 

851 metadata = exposure.getMetadata() 

852 

853 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

854 try: 

855 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

856 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

857 except Exception: 

858 self.log.warn("Could not set normalized MAGZERO in header: no " 

859 "exposure time") 

860 magZero = math.nan 

861 

862 try: 

863 metadata.set('MAGZERO', magZero) 

864 metadata.set('MAGZERO_RMS', photoRes.sigma) 

865 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

866 metadata.set('COLORTERM1', 0.0) 

867 metadata.set('COLORTERM2', 0.0) 

868 metadata.set('COLORTERM3', 0.0) 

869 except Exception as e: 

870 self.log.warn("Could not set exposure metadata: %s" % (e,)) 

871 

872 def copyIcSourceFields(self, icSourceCat, sourceCat): 

873 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

874 

875 @param[in] icSourceCat catalog from which to copy fields 

876 @param[in,out] sourceCat catalog to which to copy fields 

877 

878 The fields copied are those specified by `config.icSourceFieldsToCopy` 

879 that actually exist in the schema. This was set up by the constructor 

880 using self.schemaMapper. 

881 """ 

882 if self.schemaMapper is None: 

883 raise RuntimeError("To copy icSource fields you must specify " 

884 "icSourceSchema nd icSourceKeys when " 

885 "constructing this task") 

886 if icSourceCat is None or sourceCat is None: 

887 raise RuntimeError("icSourceCat and sourceCat must both be " 

888 "specified") 

889 if len(self.config.icSourceFieldsToCopy) == 0: 

890 self.log.warn("copyIcSourceFields doing nothing because " 

891 "icSourceFieldsToCopy is empty") 

892 return 

893 

894 mc = afwTable.MatchControl() 

895 mc.findOnlyClosest = False # return all matched objects 

896 matches = afwTable.matchXy(icSourceCat, sourceCat, 

897 self.config.matchRadiusPix, mc) 

898 if self.config.doDeblend: 

899 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

900 # if deblended, keep children 

901 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

902 

903 # Because we had to allow multiple matches to handle parents, we now 

904 # need to prune to the best matches 

905 # closest matches as a dict of icSourceCat source ID: 

906 # (icSourceCat source, sourceCat source, distance in pixels) 

907 bestMatches = {} 

908 for m0, m1, d in matches: 

909 id0 = m0.getId() 

910 match = bestMatches.get(id0) 

911 if match is None or d <= match[2]: 

912 bestMatches[id0] = (m0, m1, d) 

913 matches = list(bestMatches.values()) 

914 

915 # Check that no sourceCat sources are listed twice (we already know 

916 # that each match has a unique icSourceCat source ID, due to using 

917 # that ID as the key in bestMatches) 

918 numMatches = len(matches) 

919 numUniqueSources = len(set(m[1].getId() for m in matches)) 

920 if numUniqueSources != numMatches: 

921 self.log.warn("{} icSourceCat sources matched only {} sourceCat " 

922 "sources".format(numMatches, numUniqueSources)) 

923 

924 self.log.info("Copying flags from icSourceCat to sourceCat for " 

925 "%s sources" % (numMatches,)) 

926 

927 # For each match: set the calibSourceKey flag and copy the desired 

928 # fields 

929 for icSrc, src, d in matches: 

930 src.setFlag(self.calibSourceKey, True) 

931 # src.assign copies the footprint from icSrc, which we don't want 

932 # (DM-407) 

933 # so set icSrc's footprint to src's footprint before src.assign, 

934 # then restore it 

935 icSrcFootprint = icSrc.getFootprint() 

936 try: 

937 icSrc.setFootprint(src.getFootprint()) 

938 src.assign(icSrc, self.schemaMapper) 

939 finally: 

940 icSrc.setFootprint(icSrcFootprint)