Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import IdFactory, SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from .fakes import BaseFakeSourcesTask 

41from .photoCal import PhotoCalTask 

42 

43__all__ = ["CalibrateConfig", "CalibrateTask"] 

44 

45 

46class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={}): 

48 

49 icSourceSchema = cT.InitInput( 

50 doc="Schema produced by characterize image task, used to initialize this task", 

51 name="icSrc_schema", 

52 storageClass="SourceCatalog", 

53 ) 

54 

55 outputSchema = cT.InitOutput( 

56 doc="Schema after CalibrateTask has been initialized", 

57 name="src_schema", 

58 storageClass="SourceCatalog", 

59 ) 

60 

61 exposure = cT.Input( 

62 doc="Input image to calibrate", 

63 name="icExp", 

64 storageClass="ExposureF", 

65 dimensions=("instrument", "visit", "detector"), 

66 ) 

67 

68 background = cT.Input( 

69 doc="Backgrounds determined by characterize task", 

70 name="icExpBackground", 

71 storageClass="Background", 

72 dimensions=("instrument", "visit", "detector"), 

73 ) 

74 

75 icSourceCat = cT.Input( 

76 doc="Source catalog created by characterize task", 

77 name="icSrc", 

78 storageClass="SourceCatalog", 

79 dimensions=("instrument", "visit", "detector"), 

80 ) 

81 

82 astromRefCat = cT.PrerequisiteInput( 

83 doc="Reference catalog to use for astrometry", 

84 name="cal_ref_cat", 

85 storageClass="SimpleCatalog", 

86 dimensions=("skypix",), 

87 deferLoad=True, 

88 multiple=True, 

89 ) 

90 

91 photoRefCat = cT.PrerequisiteInput( 

92 doc="Reference catalog to use for photometric calibration", 

93 name="cal_ref_cat", 

94 storageClass="SimpleCatalog", 

95 dimensions=("skypix",), 

96 deferLoad=True, 

97 multiple=True 

98 ) 

99 

100 outputExposure = cT.Output( 

101 doc="Exposure after running calibration task", 

102 name="calexp", 

103 storageClass="ExposureF", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 

107 outputCat = cT.Output( 

108 doc="Source catalog produced in calibrate task", 

109 name="src", 

110 storageClass="SourceCatalog", 

111 dimensions=("instrument", "visit", "detector"), 

112 ) 

113 

114 outputBackground = cT.Output( 

115 doc="Background models estimated in calibration task", 

116 name="calexpBackground", 

117 storageClass="Background", 

118 dimensions=("instrument", "visit", "detector"), 

119 ) 

120 

121 matches = cT.Output( 

122 doc="Source/refObj matches from the astrometry solver", 

123 name="srcMatch", 

124 storageClass="Catalog", 

125 dimensions=("instrument", "visit", "detector"), 

126 ) 

127 

128 matchesDenormalized = cT.Output( 

129 doc="Denormalized matches from astrometry solver", 

130 name="srcMatchFull", 

131 storageClass="Catalog", 

132 dimensions=("instrument", "visit", "detector"), 

133 ) 

134 

135 def __init__(self, *, config=None): 

136 super().__init__(config=config) 

137 

138 if config.doAstrometry is False: 

139 self.prerequisiteInputs.remove("astromRefCat") 

140 if config.doPhotoCal is False: 

141 self.prerequisiteInputs.remove("photoRefCat") 

142 

143 if config.doWriteMatches is False or config.doAstrometry is False: 

144 self.outputs.remove("matches") 

145 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

146 self.outputs.remove("matchesDenormalized") 

147 

148 

149class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

150 """Config for CalibrateTask""" 

151 doWrite = pexConfig.Field( 

152 dtype=bool, 

153 default=True, 

154 doc="Save calibration results?", 

155 ) 

156 doWriteHeavyFootprintsInSources = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Include HeavyFootprint data in source table? If false then heavy " 

160 "footprints are saved as normal footprints, which saves some space" 

161 ) 

162 doWriteMatches = pexConfig.Field( 

163 dtype=bool, 

164 default=True, 

165 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

166 ) 

167 doWriteMatchesDenormalized = pexConfig.Field( 

168 dtype=bool, 

169 default=False, 

170 doc=("Write reference matches in denormalized format? " 

171 "This format uses more disk space, but is more convenient to " 

172 "read. Ignored if doWriteMatches=False or doWrite=False."), 

173 ) 

174 doAstrometry = pexConfig.Field( 

175 dtype=bool, 

176 default=True, 

177 doc="Perform astrometric calibration?", 

178 ) 

179 astromRefObjLoader = pexConfig.ConfigurableField( 

180 target=LoadIndexedReferenceObjectsTask, 

181 doc="reference object loader for astrometric calibration", 

182 ) 

183 photoRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for photometric calibration", 

186 ) 

187 astrometry = pexConfig.ConfigurableField( 

188 target=AstrometryTask, 

189 doc="Perform astrometric calibration to refine the WCS", 

190 ) 

191 requireAstrometry = pexConfig.Field( 

192 dtype=bool, 

193 default=True, 

194 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

195 "false."), 

196 ) 

197 doPhotoCal = pexConfig.Field( 

198 dtype=bool, 

199 default=True, 

200 doc="Perform phometric calibration?", 

201 ) 

202 requirePhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

206 "false."), 

207 ) 

208 photoCal = pexConfig.ConfigurableField( 

209 target=PhotoCalTask, 

210 doc="Perform photometric calibration", 

211 ) 

212 icSourceFieldsToCopy = pexConfig.ListField( 

213 dtype=str, 

214 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

215 doc=("Fields to copy from the icSource catalog to the output catalog " 

216 "for matching sources Any missing fields will trigger a " 

217 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

218 ) 

219 matchRadiusPix = pexConfig.Field( 

220 dtype=float, 

221 default=3, 

222 doc=("Match radius for matching icSourceCat objects to sourceCat " 

223 "objects (pixels)"), 

224 ) 

225 checkUnitsParseStrict = pexConfig.Field( 

226 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

227 "'warn' or 'silent'"), 

228 dtype=str, 

229 default="raise", 

230 ) 

231 detection = pexConfig.ConfigurableField( 

232 target=SourceDetectionTask, 

233 doc="Detect sources" 

234 ) 

235 doDeblend = pexConfig.Field( 

236 dtype=bool, 

237 default=True, 

238 doc="Run deblender input exposure" 

239 ) 

240 deblend = pexConfig.ConfigurableField( 

241 target=SourceDeblendTask, 

242 doc="Split blended sources into their components" 

243 ) 

244 measurement = pexConfig.ConfigurableField( 

245 target=SingleFrameMeasurementTask, 

246 doc="Measure sources" 

247 ) 

248 doApCorr = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Run subtask to apply aperture correction" 

252 ) 

253 applyApCorr = pexConfig.ConfigurableField( 

254 target=ApplyApCorrTask, 

255 doc="Subtask to apply aperture corrections" 

256 ) 

257 # If doApCorr is False, and the exposure does not have apcorrections 

258 # already applied, the active plugins in catalogCalculation almost 

259 # certainly should not contain the characterization plugin 

260 catalogCalculation = pexConfig.ConfigurableField( 

261 target=CatalogCalculationTask, 

262 doc="Subtask to run catalogCalculation plugins on catalog" 

263 ) 

264 doInsertFakes = pexConfig.Field( 

265 dtype=bool, 

266 default=False, 

267 doc="Run fake sources injection task" 

268 ) 

269 insertFakes = pexConfig.ConfigurableField( 

270 target=BaseFakeSourcesTask, 

271 doc="Injection of fake sources for testing purposes (must be " 

272 "retargeted)" 

273 ) 

274 doWriteExposure = pexConfig.Field( 

275 dtype=bool, 

276 default=True, 

277 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

278 "normal calexp but as a fakes_calexp." 

279 ) 

280 

281 def setDefaults(self): 

282 super().setDefaults() 

283 self.detection.doTempLocalBackground = False 

284 self.deblend.maxFootprintSize = 2000 

285 

286 def validate(self): 

287 super().validate() 

288 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

289 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

290 raise ValueError( 

291 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

292 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

293 ) 

294 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

295 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

296 raise ValueError( 

297 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

298 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

299 ) 

300 

301 

302## \addtogroup LSST_task_documentation 

303## \{ 

304## \page CalibrateTask 

305## \ref CalibrateTask_ "CalibrateTask" 

306## \copybrief CalibrateTask 

307## \} 

308 

309class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

310 r"""!Calibrate an exposure: measure sources and perform astrometric and 

311 photometric calibration 

312 

313 @anchor CalibrateTask_ 

314 

315 @section pipe_tasks_calibrate_Contents Contents 

316 

317 - @ref pipe_tasks_calibrate_Purpose 

318 - @ref pipe_tasks_calibrate_Initialize 

319 - @ref pipe_tasks_calibrate_IO 

320 - @ref pipe_tasks_calibrate_Config 

321 - @ref pipe_tasks_calibrate_Metadata 

322 - @ref pipe_tasks_calibrate_Debug 

323 

324 

325 @section pipe_tasks_calibrate_Purpose Description 

326 

327 Given an exposure with a good PSF model and aperture correction map 

328 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

329 operations: 

330 - Run detection and measurement 

331 - Run astrometry subtask to fit an improved WCS 

332 - Run photoCal subtask to fit the exposure's photometric zero-point 

333 

334 @section pipe_tasks_calibrate_Initialize Task initialisation 

335 

336 @copydoc \_\_init\_\_ 

337 

338 @section pipe_tasks_calibrate_IO Invoking the Task 

339 

340 If you want this task to unpersist inputs or persist outputs, then call 

341 the `runDataRef` method (a wrapper around the `run` method). 

342 

343 If you already have the inputs unpersisted and do not want to persist the 

344 output then it is more direct to call the `run` method: 

345 

346 @section pipe_tasks_calibrate_Config Configuration parameters 

347 

348 See @ref CalibrateConfig 

349 

350 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

351 

352 Exposure metadata 

353 <dl> 

354 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

355 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

356 task 

357 <dt>COLORTERM1 <dd>?? (always 0.0) 

358 <dt>COLORTERM2 <dd>?? (always 0.0) 

359 <dt>COLORTERM3 <dd>?? (always 0.0) 

360 </dl> 

361 

362 @section pipe_tasks_calibrate_Debug Debug variables 

363 

364 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

365 interface supports a flag 

366 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

367 for more about `debug.py`. 

368 

369 CalibrateTask has a debug dictionary containing one key: 

370 <dl> 

371 <dt>calibrate 

372 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

373 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

374 the meaning of the various symbols. 

375 </dl> 

376 

377 For example, put something like: 

378 @code{.py} 

379 import lsstDebug 

380 def DebugInfo(name): 

381 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

382 # call us recursively 

383 if name == "lsst.pipe.tasks.calibrate": 

384 di.display = dict( 

385 calibrate = 1, 

386 ) 

387 

388 return di 

389 

390 lsstDebug.Info = DebugInfo 

391 @endcode 

392 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

393 flag. 

394 

395 Some subtasks may have their own debug variables; see individual Task 

396 documentation. 

397 """ 

398 

399 # Example description used to live here, removed 2-20-2017 as per 

400 # https://jira.lsstcorp.org/browse/DM-9520 

401 

402 ConfigClass = CalibrateConfig 

403 _DefaultName = "calibrate" 

404 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

405 

406 def __init__(self, butler=None, astromRefObjLoader=None, 

407 photoRefObjLoader=None, icSourceSchema=None, 

408 initInputs=None, **kwargs): 

409 """!Construct a CalibrateTask 

410 

411 @param[in] butler The butler is passed to the refObjLoader constructor 

412 in case it is needed. Ignored if the refObjLoader argument 

413 provides a loader directly. 

414 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

415 that supplies an external reference catalog for astrometric 

416 calibration. May be None if the desired loader can be constructed 

417 from the butler argument or all steps requiring a reference catalog 

418 are disabled. 

419 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

420 that supplies an external reference catalog for photometric 

421 calibration. May be None if the desired loader can be constructed 

422 from the butler argument or all steps requiring a reference catalog 

423 are disabled. 

424 @param[in] icSourceSchema schema for icSource catalog, or None. 

425 Schema values specified in config.icSourceFieldsToCopy will be 

426 taken from this schema. If set to None, no values will be 

427 propagated from the icSourceCatalog 

428 @param[in,out] kwargs other keyword arguments for 

429 lsst.pipe.base.CmdLineTask 

430 """ 

431 super().__init__(**kwargs) 

432 

433 if icSourceSchema is None and butler is not None: 

434 # Use butler to read icSourceSchema from disk. 

435 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

436 

437 if icSourceSchema is None and butler is None and initInputs is not None: 

438 icSourceSchema = initInputs['icSourceSchema'].schema 

439 

440 if icSourceSchema is not None: 

441 # use a schema mapper to avoid copying each field separately 

442 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

443 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

444 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

445 

446 # Add fields to copy from an icSource catalog 

447 # and a field to indicate that the source matched a source in that 

448 # catalog. If any fields are missing then raise an exception, but 

449 # first find all missing fields in order to make the error message 

450 # more useful. 

451 self.calibSourceKey = self.schemaMapper.addOutputField( 

452 afwTable.Field["Flag"]("calib_detected", 

453 "Source was detected as an icSource")) 

454 missingFieldNames = [] 

455 for fieldName in self.config.icSourceFieldsToCopy: 

456 try: 

457 schemaItem = icSourceSchema.find(fieldName) 

458 except Exception: 

459 missingFieldNames.append(fieldName) 

460 else: 

461 # field found; if addMapping fails then raise an exception 

462 self.schemaMapper.addMapping(schemaItem.getKey()) 

463 

464 if missingFieldNames: 

465 raise RuntimeError("isSourceCat is missing fields {} " 

466 "specified in icSourceFieldsToCopy" 

467 .format(missingFieldNames)) 

468 

469 # produce a temporary schema to pass to the subtasks; finalize it 

470 # later 

471 self.schema = self.schemaMapper.editOutputSchema() 

472 else: 

473 self.schemaMapper = None 

474 self.schema = afwTable.SourceTable.makeMinimalSchema() 

475 self.makeSubtask('detection', schema=self.schema) 

476 

477 self.algMetadata = dafBase.PropertyList() 

478 

479 # Only create a subtask for fakes if configuration option is set 

480 # N.B. the config for fake object task must be retargeted to a child 

481 # of BaseFakeSourcesTask 

482 if self.config.doInsertFakes: 

483 self.makeSubtask("insertFakes") 

484 

485 if self.config.doDeblend: 

486 self.makeSubtask("deblend", schema=self.schema) 

487 self.makeSubtask('measurement', schema=self.schema, 

488 algMetadata=self.algMetadata) 

489 if self.config.doApCorr: 

490 self.makeSubtask('applyApCorr', schema=self.schema) 

491 self.makeSubtask('catalogCalculation', schema=self.schema) 

492 

493 if self.config.doAstrometry: 

494 if astromRefObjLoader is None and butler is not None: 

495 self.makeSubtask('astromRefObjLoader', butler=butler) 

496 astromRefObjLoader = self.astromRefObjLoader 

497 self.pixelMargin = astromRefObjLoader.config.pixelMargin 

498 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

499 schema=self.schema) 

500 if self.config.doPhotoCal: 

501 if photoRefObjLoader is None and butler is not None: 

502 self.makeSubtask('photoRefObjLoader', butler=butler) 

503 photoRefObjLoader = self.photoRefObjLoader 

504 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

505 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

506 schema=self.schema) 

507 

508 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

509 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

510 "reference object loaders.") 

511 

512 if self.schemaMapper is not None: 

513 # finalize the schema 

514 self.schema = self.schemaMapper.getOutputSchema() 

515 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

516 

517 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

518 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

519 self.outputSchema = sourceCatSchema 

520 

521 @pipeBase.timeMethod 

522 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

523 doUnpersist=True): 

524 """!Calibrate an exposure, optionally unpersisting inputs and 

525 persisting outputs. 

526 

527 This is a wrapper around the `run` method that unpersists inputs 

528 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

529 

530 @param[in] dataRef butler data reference corresponding to a science 

531 image 

532 @param[in,out] exposure characterized exposure (an 

533 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

534 icExp and icBackground. See `run` method for details of what is 

535 read and written. 

536 @param[in,out] background initial model of background already 

537 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

538 None if no background has been subtracted, though that is unusual 

539 for calibration. A refined background model is output. Ignored if 

540 exposure is None. 

541 @param[in] icSourceCat catalog from which to copy the fields specified 

542 by icSourceKeys, or None; 

543 @param[in] doUnpersist unpersist data: 

544 - if True, exposure, background and icSourceCat are read from 

545 dataRef and those three arguments must all be None; 

546 - if False the exposure must be provided; background and 

547 icSourceCat are optional. True is intended for running as a 

548 command-line task, False for running as a subtask 

549 @return same data as the calibrate method 

550 """ 

551 self.log.info("Processing %s" % (dataRef.dataId)) 

552 

553 if doUnpersist: 

554 if any(item is not None for item in (exposure, background, 

555 icSourceCat)): 

556 raise RuntimeError("doUnpersist true; exposure, background " 

557 "and icSourceCat must all be None") 

558 exposure = dataRef.get("icExp", immediate=True) 

559 background = dataRef.get("icExpBackground", immediate=True) 

560 icSourceCat = dataRef.get("icSrc", immediate=True) 

561 elif exposure is None: 

562 raise RuntimeError("doUnpersist false; exposure must be provided") 

563 

564 exposureIdInfo = dataRef.get("expIdInfo") 

565 

566 calRes = self.run( 

567 exposure=exposure, 

568 exposureIdInfo=exposureIdInfo, 

569 background=background, 

570 icSourceCat=icSourceCat, 

571 ) 

572 

573 if self.config.doWrite: 

574 self.writeOutputs( 

575 dataRef=dataRef, 

576 exposure=calRes.exposure, 

577 background=calRes.background, 

578 sourceCat=calRes.sourceCat, 

579 astromMatches=calRes.astromMatches, 

580 matchMeta=calRes.matchMeta, 

581 ) 

582 

583 return calRes 

584 

585 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

586 inputs = butlerQC.get(inputRefs) 

587 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

588 returnMaxBits=True) 

589 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

590 

591 if self.config.doAstrometry: 

592 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

593 for ref in inputRefs.astromRefCat], 

594 refCats=inputs.pop('astromRefCat'), 

595 config=self.config.astromRefObjLoader, log=self.log) 

596 self.pixelMargin = refObjLoader.config.pixelMargin 

597 self.astrometry.setRefObjLoader(refObjLoader) 

598 

599 if self.config.doPhotoCal: 

600 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

601 for ref in inputRefs.photoRefCat], 

602 refCats=inputs.pop('photoRefCat'), 

603 config=self.config.photoRefObjLoader, 

604 log=self.log) 

605 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

606 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

607 

608 outputs = self.run(**inputs) 

609 

610 if self.config.doWriteMatches and self.config.doAstrometry: 

611 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

612 normalizedMatches.table.setMetadata(outputs.matchMeta) 

613 if self.config.doWriteMatchesDenormalized: 

614 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

615 outputs.matchesDenormalized = denormMatches 

616 outputs.matches = normalizedMatches 

617 butlerQC.put(outputs, outputRefs) 

618 

619 def run(self, exposure, exposureIdInfo=None, background=None, 

620 icSourceCat=None): 

621 """!Calibrate an exposure (science image or coadd) 

622 

623 @param[in,out] exposure exposure to calibrate (an 

624 lsst.afw.image.ExposureF or similar); 

625 in: 

626 - MaskedImage 

627 - Psf 

628 out: 

629 - MaskedImage has background subtracted 

630 - Wcs is replaced 

631 - PhotoCalib is replaced 

632 @param[in] exposureIdInfo ID info for exposure (an 

633 lsst.obs.base.ExposureIdInfo) If not provided, returned 

634 SourceCatalog IDs will not be globally unique. 

635 @param[in,out] background background model already subtracted from 

636 exposure (an lsst.afw.math.BackgroundList). May be None if no 

637 background has been subtracted, though that is unusual for 

638 calibration. A refined background model is output. 

639 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

640 from which we can copy some fields. 

641 

642 @return pipe_base Struct containing these fields: 

643 - exposure calibrate science exposure with refined WCS and PhotoCalib 

644 - background model of background subtracted from exposure (an 

645 lsst.afw.math.BackgroundList) 

646 - sourceCat catalog of measured sources 

647 - astromMatches list of source/refObj matches from the astrometry 

648 solver 

649 """ 

650 # detect, deblend and measure sources 

651 if exposureIdInfo is None: 

652 exposureIdInfo = ExposureIdInfo() 

653 

654 if background is None: 

655 background = BackgroundList() 

656 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, 

657 exposureIdInfo.unusedBits) 

658 table = SourceTable.make(self.schema, sourceIdFactory) 

659 table.setMetadata(self.algMetadata) 

660 

661 detRes = self.detection.run(table=table, exposure=exposure, 

662 doSmooth=True) 

663 sourceCat = detRes.sources 

664 if detRes.fpSets.background: 

665 for bg in detRes.fpSets.background: 

666 background.append(bg) 

667 if self.config.doDeblend: 

668 self.deblend.run(exposure=exposure, sources=sourceCat) 

669 self.measurement.run( 

670 measCat=sourceCat, 

671 exposure=exposure, 

672 exposureId=exposureIdInfo.expId 

673 ) 

674 if self.config.doApCorr: 

675 self.applyApCorr.run( 

676 catalog=sourceCat, 

677 apCorrMap=exposure.getInfo().getApCorrMap() 

678 ) 

679 self.catalogCalculation.run(sourceCat) 

680 

681 if icSourceCat is not None and \ 

682 len(self.config.icSourceFieldsToCopy) > 0: 

683 self.copyIcSourceFields(icSourceCat=icSourceCat, 

684 sourceCat=sourceCat) 

685 

686 # TODO DM-11568: this contiguous check-and-copy could go away if we 

687 # reserve enough space during SourceDetection and/or SourceDeblend. 

688 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

689 # contiguity now, so views are preserved from here on. 

690 if not sourceCat.isContiguous(): 

691 sourceCat = sourceCat.copy(deep=True) 

692 

693 # perform astrometry calibration: 

694 # fit an improved WCS and update the exposure's WCS in place 

695 astromMatches = None 

696 matchMeta = None 

697 if self.config.doAstrometry: 

698 try: 

699 astromRes = self.astrometry.run( 

700 exposure=exposure, 

701 sourceCat=sourceCat, 

702 ) 

703 astromMatches = astromRes.matches 

704 matchMeta = astromRes.matchMeta 

705 except Exception as e: 

706 if self.config.requireAstrometry: 

707 raise 

708 self.log.warn("Unable to perform astrometric calibration " 

709 "(%s): attempting to proceed" % e) 

710 

711 # compute photometric calibration 

712 if self.config.doPhotoCal: 

713 try: 

714 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

715 exposure.setPhotoCalib(photoRes.photoCalib) 

716 # TODO: reword this to phrase it in terms of the calibration factor? 

717 self.log.info("Photometric zero-point: %f" % 

718 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

719 self.setMetadata(exposure=exposure, photoRes=photoRes) 

720 except Exception as e: 

721 if self.config.requirePhotoCal: 

722 raise 

723 self.log.warn("Unable to perform photometric calibration " 

724 "(%s): attempting to proceed" % e) 

725 self.setMetadata(exposure=exposure, photoRes=None) 

726 

727 if self.config.doInsertFakes: 

728 self.insertFakes.run(exposure, background=background) 

729 

730 table = SourceTable.make(self.schema, sourceIdFactory) 

731 table.setMetadata(self.algMetadata) 

732 

733 detRes = self.detection.run(table=table, exposure=exposure, 

734 doSmooth=True) 

735 sourceCat = detRes.sources 

736 if detRes.fpSets.background: 

737 for bg in detRes.fpSets.background: 

738 background.append(bg) 

739 if self.config.doDeblend: 

740 self.deblend.run(exposure=exposure, sources=sourceCat) 

741 self.measurement.run( 

742 measCat=sourceCat, 

743 exposure=exposure, 

744 exposureId=exposureIdInfo.expId 

745 ) 

746 if self.config.doApCorr: 

747 self.applyApCorr.run( 

748 catalog=sourceCat, 

749 apCorrMap=exposure.getInfo().getApCorrMap() 

750 ) 

751 self.catalogCalculation.run(sourceCat) 

752 

753 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

754 self.copyIcSourceFields(icSourceCat=icSourceCat, 

755 sourceCat=sourceCat) 

756 

757 frame = getDebugFrame(self._display, "calibrate") 

758 if frame: 

759 displayAstrometry( 

760 sourceCat=sourceCat, 

761 exposure=exposure, 

762 matches=astromMatches, 

763 frame=frame, 

764 pause=False, 

765 ) 

766 

767 return pipeBase.Struct( 

768 exposure=exposure, 

769 background=background, 

770 sourceCat=sourceCat, 

771 astromMatches=astromMatches, 

772 matchMeta=matchMeta, 

773 # These are duplicate entries with different names for use with 

774 # gen3 middleware 

775 outputExposure=exposure, 

776 outputCat=sourceCat, 

777 outputBackground=background, 

778 ) 

779 

780 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

781 astromMatches, matchMeta): 

782 """Write output data to the output repository 

783 

784 @param[in] dataRef butler data reference corresponding to a science 

785 image 

786 @param[in] exposure exposure to write 

787 @param[in] background background model for exposure 

788 @param[in] sourceCat catalog of measured sources 

789 @param[in] astromMatches list of source/refObj matches from the 

790 astrometry solver 

791 """ 

792 dataRef.put(sourceCat, "src") 

793 if self.config.doWriteMatches and astromMatches is not None: 

794 normalizedMatches = afwTable.packMatches(astromMatches) 

795 normalizedMatches.table.setMetadata(matchMeta) 

796 dataRef.put(normalizedMatches, "srcMatch") 

797 if self.config.doWriteMatchesDenormalized: 

798 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

799 dataRef.put(denormMatches, "srcMatchFull") 

800 if self.config.doWriteExposure: 

801 dataRef.put(exposure, "calexp") 

802 dataRef.put(background, "calexpBackground") 

803 

804 def getSchemaCatalogs(self): 

805 """Return a dict of empty catalogs for each catalog dataset produced 

806 by this task. 

807 """ 

808 sourceCat = afwTable.SourceCatalog(self.schema) 

809 sourceCat.getTable().setMetadata(self.algMetadata) 

810 return {"src": sourceCat} 

811 

812 def setMetadata(self, exposure, photoRes=None): 

813 """!Set task and exposure metadata 

814 

815 Logs a warning and continues if needed data is missing. 

816 

817 @param[in,out] exposure exposure whose metadata is to be set 

818 @param[in] photoRes results of running photoCal; if None then it was 

819 not run 

820 """ 

821 if photoRes is None: 

822 return 

823 

824 metadata = exposure.getMetadata() 

825 

826 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

827 try: 

828 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

829 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

830 except Exception: 

831 self.log.warn("Could not set normalized MAGZERO in header: no " 

832 "exposure time") 

833 magZero = math.nan 

834 

835 try: 

836 metadata.set('MAGZERO', magZero) 

837 metadata.set('MAGZERO_RMS', photoRes.sigma) 

838 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

839 metadata.set('COLORTERM1', 0.0) 

840 metadata.set('COLORTERM2', 0.0) 

841 metadata.set('COLORTERM3', 0.0) 

842 except Exception as e: 

843 self.log.warn("Could not set exposure metadata: %s" % (e,)) 

844 

845 def copyIcSourceFields(self, icSourceCat, sourceCat): 

846 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

847 

848 @param[in] icSourceCat catalog from which to copy fields 

849 @param[in,out] sourceCat catalog to which to copy fields 

850 

851 The fields copied are those specified by `config.icSourceFieldsToCopy` 

852 that actually exist in the schema. This was set up by the constructor 

853 using self.schemaMapper. 

854 """ 

855 if self.schemaMapper is None: 

856 raise RuntimeError("To copy icSource fields you must specify " 

857 "icSourceSchema nd icSourceKeys when " 

858 "constructing this task") 

859 if icSourceCat is None or sourceCat is None: 

860 raise RuntimeError("icSourceCat and sourceCat must both be " 

861 "specified") 

862 if len(self.config.icSourceFieldsToCopy) == 0: 

863 self.log.warn("copyIcSourceFields doing nothing because " 

864 "icSourceFieldsToCopy is empty") 

865 return 

866 

867 mc = afwTable.MatchControl() 

868 mc.findOnlyClosest = False # return all matched objects 

869 matches = afwTable.matchXy(icSourceCat, sourceCat, 

870 self.config.matchRadiusPix, mc) 

871 if self.config.doDeblend: 

872 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

873 # if deblended, keep children 

874 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

875 

876 # Because we had to allow multiple matches to handle parents, we now 

877 # need to prune to the best matches 

878 # closest matches as a dict of icSourceCat source ID: 

879 # (icSourceCat source, sourceCat source, distance in pixels) 

880 bestMatches = {} 

881 for m0, m1, d in matches: 

882 id0 = m0.getId() 

883 match = bestMatches.get(id0) 

884 if match is None or d <= match[2]: 

885 bestMatches[id0] = (m0, m1, d) 

886 matches = list(bestMatches.values()) 

887 

888 # Check that no sourceCat sources are listed twice (we already know 

889 # that each match has a unique icSourceCat source ID, due to using 

890 # that ID as the key in bestMatches) 

891 numMatches = len(matches) 

892 numUniqueSources = len(set(m[1].getId() for m in matches)) 

893 if numUniqueSources != numMatches: 

894 self.log.warn("{} icSourceCat sources matched only {} sourceCat " 

895 "sources".format(numMatches, numUniqueSources)) 

896 

897 self.log.info("Copying flags from icSourceCat to sourceCat for " 

898 "%s sources" % (numMatches,)) 

899 

900 # For each match: set the calibSourceKey flag and copy the desired 

901 # fields 

902 for icSrc, src, d in matches: 

903 src.setFlag(self.calibSourceKey, True) 

904 # src.assign copies the footprint from icSrc, which we don't want 

905 # (DM-407) 

906 # so set icSrc's footprint to src's footprint before src.assign, 

907 # then restore it 

908 icSrcFootprint = icSrc.getFootprint() 

909 try: 

910 icSrc.setFootprint(src.getFootprint()) 

911 src.assign(icSrc, self.schemaMapper) 

912 finally: 

913 icSrc.setFootprint(icSrcFootprint)