Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import IdFactory, SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from .fakes import BaseFakeSourcesTask 

41from .photoCal import PhotoCalTask 

42 

43__all__ = ["CalibrateConfig", "CalibrateTask"] 

44 

45 

46class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={}): 

48 

49 icSourceSchema = cT.InitInput( 

50 doc="Schema produced by characterize image task, used to initialize this task", 

51 name="icSrc_schema", 

52 storageClass="SourceCatalog", 

53 ) 

54 

55 outputSchema = cT.InitOutput( 

56 doc="Schema after CalibrateTask has been initialized", 

57 name="src_schema", 

58 storageClass="SourceCatalog", 

59 ) 

60 

61 exposure = cT.Input( 

62 doc="Input image to calibrate", 

63 name="icExp", 

64 storageClass="ExposureF", 

65 dimensions=("instrument", "visit", "detector"), 

66 ) 

67 

68 background = cT.Input( 

69 doc="Backgrounds determined by characterize task", 

70 name="icExpBackground", 

71 storageClass="Background", 

72 dimensions=("instrument", "visit", "detector"), 

73 ) 

74 

75 icSourceCat = cT.Input( 

76 doc="Source catalog created by characterize task", 

77 name="icSrc", 

78 storageClass="SourceCatalog", 

79 dimensions=("instrument", "visit", "detector"), 

80 ) 

81 

82 astromRefCat = cT.PrerequisiteInput( 

83 doc="Reference catalog to use for astrometry", 

84 name="cal_ref_cat", 

85 storageClass="SimpleCatalog", 

86 dimensions=("skypix",), 

87 deferLoad=True, 

88 multiple=True, 

89 ) 

90 

91 photoRefCat = cT.PrerequisiteInput( 

92 doc="Reference catalog to use for photometric calibration", 

93 name="cal_ref_cat", 

94 storageClass="SimpleCatalog", 

95 dimensions=("skypix",), 

96 deferLoad=True, 

97 multiple=True 

98 ) 

99 

100 outputExposure = cT.Output( 

101 doc="Exposure after running calibration task", 

102 name="calexp", 

103 storageClass="ExposureF", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 

107 outputCat = cT.Output( 

108 doc="Source catalog produced in calibrate task", 

109 name="src", 

110 storageClass="SourceCatalog", 

111 dimensions=("instrument", "visit", "detector"), 

112 ) 

113 

114 outputBackground = cT.Output( 

115 doc="Background models estimated in calibration task", 

116 name="calexpBackground", 

117 storageClass="Background", 

118 dimensions=("instrument", "visit", "detector"), 

119 ) 

120 

121 matches = cT.Output( 

122 doc="Source/refObj matches from the astrometry solver", 

123 name="srcMatch", 

124 storageClass="Catalog", 

125 dimensions=("instrument", "visit", "detector"), 

126 ) 

127 

128 matchesDenormalized = cT.Output( 

129 doc="Denormalized matches from astrometry solver", 

130 name="srcMatchFull", 

131 storageClass="Catalog", 

132 dimensions=("instrument", "visit", "detector"), 

133 ) 

134 

135 def __init__(self, *, config=None): 

136 super().__init__(config=config) 

137 

138 if config.doAstrometry is False: 

139 self.prerequisiteInputs.remove("astromRefCat") 

140 if config.doPhotoCal is False: 

141 self.prerequisiteInputs.remove("photoRefCat") 

142 

143 if config.doWriteMatches is False or config.doAstrometry is False: 

144 self.outputs.remove("matches") 

145 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

146 self.outputs.remove("matchesDenormalized") 

147 

148 

149class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

150 """Config for CalibrateTask""" 

151 doWrite = pexConfig.Field( 

152 dtype=bool, 

153 default=True, 

154 doc="Save calibration results?", 

155 ) 

156 doWriteHeavyFootprintsInSources = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Include HeavyFootprint data in source table? If false then heavy " 

160 "footprints are saved as normal footprints, which saves some space" 

161 ) 

162 doWriteMatches = pexConfig.Field( 

163 dtype=bool, 

164 default=True, 

165 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

166 ) 

167 doWriteMatchesDenormalized = pexConfig.Field( 

168 dtype=bool, 

169 default=False, 

170 doc=("Write reference matches in denormalized format? " 

171 "This format uses more disk space, but is more convenient to " 

172 "read. Ignored if doWriteMatches=False or doWrite=False."), 

173 ) 

174 doAstrometry = pexConfig.Field( 

175 dtype=bool, 

176 default=True, 

177 doc="Perform astrometric calibration?", 

178 ) 

179 astromRefObjLoader = pexConfig.ConfigurableField( 

180 target=LoadIndexedReferenceObjectsTask, 

181 doc="reference object loader for astrometric calibration", 

182 ) 

183 photoRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for photometric calibration", 

186 ) 

187 astrometry = pexConfig.ConfigurableField( 

188 target=AstrometryTask, 

189 doc="Perform astrometric calibration to refine the WCS", 

190 ) 

191 requireAstrometry = pexConfig.Field( 

192 dtype=bool, 

193 default=True, 

194 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

195 "false."), 

196 ) 

197 doPhotoCal = pexConfig.Field( 

198 dtype=bool, 

199 default=True, 

200 doc="Perform phometric calibration?", 

201 ) 

202 requirePhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

206 "false."), 

207 ) 

208 photoCal = pexConfig.ConfigurableField( 

209 target=PhotoCalTask, 

210 doc="Perform photometric calibration", 

211 ) 

212 icSourceFieldsToCopy = pexConfig.ListField( 

213 dtype=str, 

214 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

215 doc=("Fields to copy from the icSource catalog to the output catalog " 

216 "for matching sources Any missing fields will trigger a " 

217 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

218 ) 

219 matchRadiusPix = pexConfig.Field( 

220 dtype=float, 

221 default=3, 

222 doc=("Match radius for matching icSourceCat objects to sourceCat " 

223 "objects (pixels)"), 

224 ) 

225 checkUnitsParseStrict = pexConfig.Field( 

226 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

227 "'warn' or 'silent'"), 

228 dtype=str, 

229 default="raise", 

230 ) 

231 detection = pexConfig.ConfigurableField( 

232 target=SourceDetectionTask, 

233 doc="Detect sources" 

234 ) 

235 doDeblend = pexConfig.Field( 

236 dtype=bool, 

237 default=True, 

238 doc="Run deblender input exposure" 

239 ) 

240 deblend = pexConfig.ConfigurableField( 

241 target=SourceDeblendTask, 

242 doc="Split blended sources into their components" 

243 ) 

244 measurement = pexConfig.ConfigurableField( 

245 target=SingleFrameMeasurementTask, 

246 doc="Measure sources" 

247 ) 

248 doApCorr = pexConfig.Field( 

249 dtype=bool, 

250 default=True, 

251 doc="Run subtask to apply aperture correction" 

252 ) 

253 applyApCorr = pexConfig.ConfigurableField( 

254 target=ApplyApCorrTask, 

255 doc="Subtask to apply aperture corrections" 

256 ) 

257 # If doApCorr is False, and the exposure does not have apcorrections 

258 # already applied, the active plugins in catalogCalculation almost 

259 # certainly should not contain the characterization plugin 

260 catalogCalculation = pexConfig.ConfigurableField( 

261 target=CatalogCalculationTask, 

262 doc="Subtask to run catalogCalculation plugins on catalog" 

263 ) 

264 doInsertFakes = pexConfig.Field( 

265 dtype=bool, 

266 default=False, 

267 doc="Run fake sources injection task" 

268 ) 

269 insertFakes = pexConfig.ConfigurableField( 

270 target=BaseFakeSourcesTask, 

271 doc="Injection of fake sources for testing purposes (must be " 

272 "retargeted)" 

273 ) 

274 doWriteExposure = pexConfig.Field( 

275 dtype=bool, 

276 default=True, 

277 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

278 "normal calexp but as a fakes_calexp." 

279 ) 

280 

281 def setDefaults(self): 

282 super().setDefaults() 

283 self.detection.doTempLocalBackground = False 

284 self.deblend.maxFootprintSize = 2000 

285 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"] 

286 

287 def validate(self): 

288 super().validate() 

289 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

290 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

291 raise ValueError( 

292 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

293 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

294 ) 

295 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

296 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

297 raise ValueError( 

298 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

299 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

300 ) 

301 

302 

303## \addtogroup LSST_task_documentation 

304## \{ 

305## \page CalibrateTask 

306## \ref CalibrateTask_ "CalibrateTask" 

307## \copybrief CalibrateTask 

308## \} 

309 

310class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

311 r"""!Calibrate an exposure: measure sources and perform astrometric and 

312 photometric calibration 

313 

314 @anchor CalibrateTask_ 

315 

316 @section pipe_tasks_calibrate_Contents Contents 

317 

318 - @ref pipe_tasks_calibrate_Purpose 

319 - @ref pipe_tasks_calibrate_Initialize 

320 - @ref pipe_tasks_calibrate_IO 

321 - @ref pipe_tasks_calibrate_Config 

322 - @ref pipe_tasks_calibrate_Metadata 

323 - @ref pipe_tasks_calibrate_Debug 

324 

325 

326 @section pipe_tasks_calibrate_Purpose Description 

327 

328 Given an exposure with a good PSF model and aperture correction map 

329 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

330 operations: 

331 - Run detection and measurement 

332 - Run astrometry subtask to fit an improved WCS 

333 - Run photoCal subtask to fit the exposure's photometric zero-point 

334 

335 @section pipe_tasks_calibrate_Initialize Task initialisation 

336 

337 @copydoc \_\_init\_\_ 

338 

339 @section pipe_tasks_calibrate_IO Invoking the Task 

340 

341 If you want this task to unpersist inputs or persist outputs, then call 

342 the `runDataRef` method (a wrapper around the `run` method). 

343 

344 If you already have the inputs unpersisted and do not want to persist the 

345 output then it is more direct to call the `run` method: 

346 

347 @section pipe_tasks_calibrate_Config Configuration parameters 

348 

349 See @ref CalibrateConfig 

350 

351 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

352 

353 Exposure metadata 

354 <dl> 

355 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

356 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

357 task 

358 <dt>COLORTERM1 <dd>?? (always 0.0) 

359 <dt>COLORTERM2 <dd>?? (always 0.0) 

360 <dt>COLORTERM3 <dd>?? (always 0.0) 

361 </dl> 

362 

363 @section pipe_tasks_calibrate_Debug Debug variables 

364 

365 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

366 interface supports a flag 

367 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

368 for more about `debug.py`. 

369 

370 CalibrateTask has a debug dictionary containing one key: 

371 <dl> 

372 <dt>calibrate 

373 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

374 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

375 the meaning of the various symbols. 

376 </dl> 

377 

378 For example, put something like: 

379 @code{.py} 

380 import lsstDebug 

381 def DebugInfo(name): 

382 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

383 # call us recursively 

384 if name == "lsst.pipe.tasks.calibrate": 

385 di.display = dict( 

386 calibrate = 1, 

387 ) 

388 

389 return di 

390 

391 lsstDebug.Info = DebugInfo 

392 @endcode 

393 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

394 flag. 

395 

396 Some subtasks may have their own debug variables; see individual Task 

397 documentation. 

398 """ 

399 

400 # Example description used to live here, removed 2-20-2017 as per 

401 # https://jira.lsstcorp.org/browse/DM-9520 

402 

403 ConfigClass = CalibrateConfig 

404 _DefaultName = "calibrate" 

405 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

406 

407 def __init__(self, butler=None, astromRefObjLoader=None, 

408 photoRefObjLoader=None, icSourceSchema=None, 

409 initInputs=None, **kwargs): 

410 """!Construct a CalibrateTask 

411 

412 @param[in] butler The butler is passed to the refObjLoader constructor 

413 in case it is needed. Ignored if the refObjLoader argument 

414 provides a loader directly. 

415 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

416 that supplies an external reference catalog for astrometric 

417 calibration. May be None if the desired loader can be constructed 

418 from the butler argument or all steps requiring a reference catalog 

419 are disabled. 

420 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

421 that supplies an external reference catalog for photometric 

422 calibration. May be None if the desired loader can be constructed 

423 from the butler argument or all steps requiring a reference catalog 

424 are disabled. 

425 @param[in] icSourceSchema schema for icSource catalog, or None. 

426 Schema values specified in config.icSourceFieldsToCopy will be 

427 taken from this schema. If set to None, no values will be 

428 propagated from the icSourceCatalog 

429 @param[in,out] kwargs other keyword arguments for 

430 lsst.pipe.base.CmdLineTask 

431 """ 

432 super().__init__(**kwargs) 

433 

434 if icSourceSchema is None and butler is not None: 

435 # Use butler to read icSourceSchema from disk. 

436 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

437 

438 if icSourceSchema is None and butler is None and initInputs is not None: 

439 icSourceSchema = initInputs['icSourceSchema'].schema 

440 

441 if icSourceSchema is not None: 

442 # use a schema mapper to avoid copying each field separately 

443 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

444 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

445 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

446 

447 # Add fields to copy from an icSource catalog 

448 # and a field to indicate that the source matched a source in that 

449 # catalog. If any fields are missing then raise an exception, but 

450 # first find all missing fields in order to make the error message 

451 # more useful. 

452 self.calibSourceKey = self.schemaMapper.addOutputField( 

453 afwTable.Field["Flag"]("calib_detected", 

454 "Source was detected as an icSource")) 

455 missingFieldNames = [] 

456 for fieldName in self.config.icSourceFieldsToCopy: 

457 try: 

458 schemaItem = icSourceSchema.find(fieldName) 

459 except Exception: 

460 missingFieldNames.append(fieldName) 

461 else: 

462 # field found; if addMapping fails then raise an exception 

463 self.schemaMapper.addMapping(schemaItem.getKey()) 

464 

465 if missingFieldNames: 

466 raise RuntimeError("isSourceCat is missing fields {} " 

467 "specified in icSourceFieldsToCopy" 

468 .format(missingFieldNames)) 

469 

470 # produce a temporary schema to pass to the subtasks; finalize it 

471 # later 

472 self.schema = self.schemaMapper.editOutputSchema() 

473 else: 

474 self.schemaMapper = None 

475 self.schema = afwTable.SourceTable.makeMinimalSchema() 

476 self.makeSubtask('detection', schema=self.schema) 

477 

478 self.algMetadata = dafBase.PropertyList() 

479 

480 # Only create a subtask for fakes if configuration option is set 

481 # N.B. the config for fake object task must be retargeted to a child 

482 # of BaseFakeSourcesTask 

483 if self.config.doInsertFakes: 

484 self.makeSubtask("insertFakes") 

485 

486 if self.config.doDeblend: 

487 self.makeSubtask("deblend", schema=self.schema) 

488 self.makeSubtask('measurement', schema=self.schema, 

489 algMetadata=self.algMetadata) 

490 if self.config.doApCorr: 

491 self.makeSubtask('applyApCorr', schema=self.schema) 

492 self.makeSubtask('catalogCalculation', schema=self.schema) 

493 

494 if self.config.doAstrometry: 

495 if astromRefObjLoader is None and butler is not None: 

496 self.makeSubtask('astromRefObjLoader', butler=butler) 

497 astromRefObjLoader = self.astromRefObjLoader 

498 self.pixelMargin = astromRefObjLoader.config.pixelMargin 

499 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

500 schema=self.schema) 

501 if self.config.doPhotoCal: 

502 if photoRefObjLoader is None and butler is not None: 

503 self.makeSubtask('photoRefObjLoader', butler=butler) 

504 photoRefObjLoader = self.photoRefObjLoader 

505 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

506 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

507 schema=self.schema) 

508 

509 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

510 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

511 "reference object loaders.") 

512 

513 if self.schemaMapper is not None: 

514 # finalize the schema 

515 self.schema = self.schemaMapper.getOutputSchema() 

516 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

517 

518 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

519 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

520 self.outputSchema = sourceCatSchema 

521 

522 @pipeBase.timeMethod 

523 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

524 doUnpersist=True): 

525 """!Calibrate an exposure, optionally unpersisting inputs and 

526 persisting outputs. 

527 

528 This is a wrapper around the `run` method that unpersists inputs 

529 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

530 

531 @param[in] dataRef butler data reference corresponding to a science 

532 image 

533 @param[in,out] exposure characterized exposure (an 

534 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

535 icExp and icBackground. See `run` method for details of what is 

536 read and written. 

537 @param[in,out] background initial model of background already 

538 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

539 None if no background has been subtracted, though that is unusual 

540 for calibration. A refined background model is output. Ignored if 

541 exposure is None. 

542 @param[in] icSourceCat catalog from which to copy the fields specified 

543 by icSourceKeys, or None; 

544 @param[in] doUnpersist unpersist data: 

545 - if True, exposure, background and icSourceCat are read from 

546 dataRef and those three arguments must all be None; 

547 - if False the exposure must be provided; background and 

548 icSourceCat are optional. True is intended for running as a 

549 command-line task, False for running as a subtask 

550 @return same data as the calibrate method 

551 """ 

552 self.log.info("Processing %s" % (dataRef.dataId)) 

553 

554 if doUnpersist: 

555 if any(item is not None for item in (exposure, background, 

556 icSourceCat)): 

557 raise RuntimeError("doUnpersist true; exposure, background " 

558 "and icSourceCat must all be None") 

559 exposure = dataRef.get("icExp", immediate=True) 

560 background = dataRef.get("icExpBackground", immediate=True) 

561 icSourceCat = dataRef.get("icSrc", immediate=True) 

562 elif exposure is None: 

563 raise RuntimeError("doUnpersist false; exposure must be provided") 

564 

565 exposureIdInfo = dataRef.get("expIdInfo") 

566 

567 calRes = self.run( 

568 exposure=exposure, 

569 exposureIdInfo=exposureIdInfo, 

570 background=background, 

571 icSourceCat=icSourceCat, 

572 ) 

573 

574 if self.config.doWrite: 

575 self.writeOutputs( 

576 dataRef=dataRef, 

577 exposure=calRes.exposure, 

578 background=calRes.background, 

579 sourceCat=calRes.sourceCat, 

580 astromMatches=calRes.astromMatches, 

581 matchMeta=calRes.matchMeta, 

582 ) 

583 

584 return calRes 

585 

586 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

587 inputs = butlerQC.get(inputRefs) 

588 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

589 returnMaxBits=True) 

590 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

591 

592 if self.config.doAstrometry: 

593 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

594 for ref in inputRefs.astromRefCat], 

595 refCats=inputs.pop('astromRefCat'), 

596 config=self.config.astromRefObjLoader, log=self.log) 

597 self.pixelMargin = refObjLoader.config.pixelMargin 

598 self.astrometry.setRefObjLoader(refObjLoader) 

599 

600 if self.config.doPhotoCal: 

601 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

602 for ref in inputRefs.photoRefCat], 

603 refCats=inputs.pop('photoRefCat'), 

604 config=self.config.photoRefObjLoader, 

605 log=self.log) 

606 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

607 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

608 

609 outputs = self.run(**inputs) 

610 

611 if self.config.doWriteMatches and self.config.doAstrometry: 

612 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

613 normalizedMatches.table.setMetadata(outputs.matchMeta) 

614 if self.config.doWriteMatchesDenormalized: 

615 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

616 outputs.matchesDenormalized = denormMatches 

617 outputs.matches = normalizedMatches 

618 butlerQC.put(outputs, outputRefs) 

619 

620 def run(self, exposure, exposureIdInfo=None, background=None, 

621 icSourceCat=None): 

622 """!Calibrate an exposure (science image or coadd) 

623 

624 @param[in,out] exposure exposure to calibrate (an 

625 lsst.afw.image.ExposureF or similar); 

626 in: 

627 - MaskedImage 

628 - Psf 

629 out: 

630 - MaskedImage has background subtracted 

631 - Wcs is replaced 

632 - PhotoCalib is replaced 

633 @param[in] exposureIdInfo ID info for exposure (an 

634 lsst.obs.base.ExposureIdInfo) If not provided, returned 

635 SourceCatalog IDs will not be globally unique. 

636 @param[in,out] background background model already subtracted from 

637 exposure (an lsst.afw.math.BackgroundList). May be None if no 

638 background has been subtracted, though that is unusual for 

639 calibration. A refined background model is output. 

640 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

641 from which we can copy some fields. 

642 

643 @return pipe_base Struct containing these fields: 

644 - exposure calibrate science exposure with refined WCS and PhotoCalib 

645 - background model of background subtracted from exposure (an 

646 lsst.afw.math.BackgroundList) 

647 - sourceCat catalog of measured sources 

648 - astromMatches list of source/refObj matches from the astrometry 

649 solver 

650 """ 

651 # detect, deblend and measure sources 

652 if exposureIdInfo is None: 

653 exposureIdInfo = ExposureIdInfo() 

654 

655 if background is None: 

656 background = BackgroundList() 

657 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, 

658 exposureIdInfo.unusedBits) 

659 table = SourceTable.make(self.schema, sourceIdFactory) 

660 table.setMetadata(self.algMetadata) 

661 

662 detRes = self.detection.run(table=table, exposure=exposure, 

663 doSmooth=True) 

664 sourceCat = detRes.sources 

665 if detRes.fpSets.background: 

666 for bg in detRes.fpSets.background: 

667 background.append(bg) 

668 if self.config.doDeblend: 

669 self.deblend.run(exposure=exposure, sources=sourceCat) 

670 self.measurement.run( 

671 measCat=sourceCat, 

672 exposure=exposure, 

673 exposureId=exposureIdInfo.expId 

674 ) 

675 if self.config.doApCorr: 

676 self.applyApCorr.run( 

677 catalog=sourceCat, 

678 apCorrMap=exposure.getInfo().getApCorrMap() 

679 ) 

680 self.catalogCalculation.run(sourceCat) 

681 

682 if icSourceCat is not None and \ 

683 len(self.config.icSourceFieldsToCopy) > 0: 

684 self.copyIcSourceFields(icSourceCat=icSourceCat, 

685 sourceCat=sourceCat) 

686 

687 # TODO DM-11568: this contiguous check-and-copy could go away if we 

688 # reserve enough space during SourceDetection and/or SourceDeblend. 

689 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

690 # contiguity now, so views are preserved from here on. 

691 if not sourceCat.isContiguous(): 

692 sourceCat = sourceCat.copy(deep=True) 

693 

694 # perform astrometry calibration: 

695 # fit an improved WCS and update the exposure's WCS in place 

696 astromMatches = None 

697 matchMeta = None 

698 if self.config.doAstrometry: 

699 try: 

700 astromRes = self.astrometry.run( 

701 exposure=exposure, 

702 sourceCat=sourceCat, 

703 ) 

704 astromMatches = astromRes.matches 

705 matchMeta = astromRes.matchMeta 

706 except Exception as e: 

707 if self.config.requireAstrometry: 

708 raise 

709 self.log.warn("Unable to perform astrometric calibration " 

710 "(%s): attempting to proceed" % e) 

711 

712 # compute photometric calibration 

713 if self.config.doPhotoCal: 

714 try: 

715 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

716 exposure.setPhotoCalib(photoRes.photoCalib) 

717 # TODO: reword this to phrase it in terms of the calibration factor? 

718 self.log.info("Photometric zero-point: %f" % 

719 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

720 self.setMetadata(exposure=exposure, photoRes=photoRes) 

721 except Exception as e: 

722 if self.config.requirePhotoCal: 

723 raise 

724 self.log.warn("Unable to perform photometric calibration " 

725 "(%s): attempting to proceed" % e) 

726 self.setMetadata(exposure=exposure, photoRes=None) 

727 

728 if self.config.doInsertFakes: 

729 self.insertFakes.run(exposure, background=background) 

730 

731 table = SourceTable.make(self.schema, sourceIdFactory) 

732 table.setMetadata(self.algMetadata) 

733 

734 detRes = self.detection.run(table=table, exposure=exposure, 

735 doSmooth=True) 

736 sourceCat = detRes.sources 

737 if detRes.fpSets.background: 

738 for bg in detRes.fpSets.background: 

739 background.append(bg) 

740 if self.config.doDeblend: 

741 self.deblend.run(exposure=exposure, sources=sourceCat) 

742 self.measurement.run( 

743 measCat=sourceCat, 

744 exposure=exposure, 

745 exposureId=exposureIdInfo.expId 

746 ) 

747 if self.config.doApCorr: 

748 self.applyApCorr.run( 

749 catalog=sourceCat, 

750 apCorrMap=exposure.getInfo().getApCorrMap() 

751 ) 

752 self.catalogCalculation.run(sourceCat) 

753 

754 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

755 self.copyIcSourceFields(icSourceCat=icSourceCat, 

756 sourceCat=sourceCat) 

757 

758 frame = getDebugFrame(self._display, "calibrate") 

759 if frame: 

760 displayAstrometry( 

761 sourceCat=sourceCat, 

762 exposure=exposure, 

763 matches=astromMatches, 

764 frame=frame, 

765 pause=False, 

766 ) 

767 

768 return pipeBase.Struct( 

769 exposure=exposure, 

770 background=background, 

771 sourceCat=sourceCat, 

772 astromMatches=astromMatches, 

773 matchMeta=matchMeta, 

774 # These are duplicate entries with different names for use with 

775 # gen3 middleware 

776 outputExposure=exposure, 

777 outputCat=sourceCat, 

778 outputBackground=background, 

779 ) 

780 

781 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

782 astromMatches, matchMeta): 

783 """Write output data to the output repository 

784 

785 @param[in] dataRef butler data reference corresponding to a science 

786 image 

787 @param[in] exposure exposure to write 

788 @param[in] background background model for exposure 

789 @param[in] sourceCat catalog of measured sources 

790 @param[in] astromMatches list of source/refObj matches from the 

791 astrometry solver 

792 """ 

793 dataRef.put(sourceCat, "src") 

794 if self.config.doWriteMatches and astromMatches is not None: 

795 normalizedMatches = afwTable.packMatches(astromMatches) 

796 normalizedMatches.table.setMetadata(matchMeta) 

797 dataRef.put(normalizedMatches, "srcMatch") 

798 if self.config.doWriteMatchesDenormalized: 

799 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

800 dataRef.put(denormMatches, "srcMatchFull") 

801 if self.config.doWriteExposure: 

802 dataRef.put(exposure, "calexp") 

803 dataRef.put(background, "calexpBackground") 

804 

805 def getSchemaCatalogs(self): 

806 """Return a dict of empty catalogs for each catalog dataset produced 

807 by this task. 

808 """ 

809 sourceCat = afwTable.SourceCatalog(self.schema) 

810 sourceCat.getTable().setMetadata(self.algMetadata) 

811 return {"src": sourceCat} 

812 

813 def setMetadata(self, exposure, photoRes=None): 

814 """!Set task and exposure metadata 

815 

816 Logs a warning and continues if needed data is missing. 

817 

818 @param[in,out] exposure exposure whose metadata is to be set 

819 @param[in] photoRes results of running photoCal; if None then it was 

820 not run 

821 """ 

822 if photoRes is None: 

823 return 

824 

825 metadata = exposure.getMetadata() 

826 

827 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

828 try: 

829 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

830 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

831 except Exception: 

832 self.log.warn("Could not set normalized MAGZERO in header: no " 

833 "exposure time") 

834 magZero = math.nan 

835 

836 try: 

837 metadata.set('MAGZERO', magZero) 

838 metadata.set('MAGZERO_RMS', photoRes.sigma) 

839 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

840 metadata.set('COLORTERM1', 0.0) 

841 metadata.set('COLORTERM2', 0.0) 

842 metadata.set('COLORTERM3', 0.0) 

843 except Exception as e: 

844 self.log.warn("Could not set exposure metadata: %s" % (e,)) 

845 

846 def copyIcSourceFields(self, icSourceCat, sourceCat): 

847 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

848 

849 @param[in] icSourceCat catalog from which to copy fields 

850 @param[in,out] sourceCat catalog to which to copy fields 

851 

852 The fields copied are those specified by `config.icSourceFieldsToCopy` 

853 that actually exist in the schema. This was set up by the constructor 

854 using self.schemaMapper. 

855 """ 

856 if self.schemaMapper is None: 

857 raise RuntimeError("To copy icSource fields you must specify " 

858 "icSourceSchema nd icSourceKeys when " 

859 "constructing this task") 

860 if icSourceCat is None or sourceCat is None: 

861 raise RuntimeError("icSourceCat and sourceCat must both be " 

862 "specified") 

863 if len(self.config.icSourceFieldsToCopy) == 0: 

864 self.log.warn("copyIcSourceFields doing nothing because " 

865 "icSourceFieldsToCopy is empty") 

866 return 

867 

868 mc = afwTable.MatchControl() 

869 mc.findOnlyClosest = False # return all matched objects 

870 matches = afwTable.matchXy(icSourceCat, sourceCat, 

871 self.config.matchRadiusPix, mc) 

872 if self.config.doDeblend: 

873 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

874 # if deblended, keep children 

875 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

876 

877 # Because we had to allow multiple matches to handle parents, we now 

878 # need to prune to the best matches 

879 # closest matches as a dict of icSourceCat source ID: 

880 # (icSourceCat source, sourceCat source, distance in pixels) 

881 bestMatches = {} 

882 for m0, m1, d in matches: 

883 id0 = m0.getId() 

884 match = bestMatches.get(id0) 

885 if match is None or d <= match[2]: 

886 bestMatches[id0] = (m0, m1, d) 

887 matches = list(bestMatches.values()) 

888 

889 # Check that no sourceCat sources are listed twice (we already know 

890 # that each match has a unique icSourceCat source ID, due to using 

891 # that ID as the key in bestMatches) 

892 numMatches = len(matches) 

893 numUniqueSources = len(set(m[1].getId() for m in matches)) 

894 if numUniqueSources != numMatches: 

895 self.log.warn("{} icSourceCat sources matched only {} sourceCat " 

896 "sources".format(numMatches, numUniqueSources)) 

897 

898 self.log.info("Copying flags from icSourceCat to sourceCat for " 

899 "%s sources" % (numMatches,)) 

900 

901 # For each match: set the calibSourceKey flag and copy the desired 

902 # fields 

903 for icSrc, src, d in matches: 

904 src.setFlag(self.calibSourceKey, True) 

905 # src.assign copies the footprint from icSrc, which we don't want 

906 # (DM-407) 

907 # so set icSrc's footprint to src's footprint before src.assign, 

908 # then restore it 

909 icSrcFootprint = icSrc.getFootprint() 

910 try: 

911 icSrc.setFootprint(src.getFootprint()) 

912 src.assign(icSrc, self.schemaMapper) 

913 finally: 

914 icSrc.setFootprint(icSrcFootprint)