Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import IdFactory, SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from .fakes import BaseFakeSourcesTask 

41from .photoCal import PhotoCalTask 

42 

43__all__ = ["CalibrateConfig", "CalibrateTask"] 

44 

45 

46class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={}): 

48 

49 icSourceSchema = cT.InitInput( 

50 doc="Schema produced by characterize image task, used to initialize this task", 

51 name="icSrc_schema", 

52 storageClass="SourceCatalog", 

53 ) 

54 

55 outputSchema = cT.InitOutput( 

56 doc="Schema after CalibrateTask has been initialized", 

57 name="src_schema", 

58 storageClass="SourceCatalog", 

59 ) 

60 

61 exposure = cT.Input( 

62 doc="Input image to calibrate", 

63 name="icExp", 

64 storageClass="ExposureF", 

65 dimensions=("instrument", "visit", "detector"), 

66 ) 

67 

68 background = cT.Input( 

69 doc="Backgrounds determined by characterize task", 

70 name="icExpBackground", 

71 storageClass="Background", 

72 dimensions=("instrument", "visit", "detector"), 

73 ) 

74 

75 icSourceCat = cT.Input( 

76 doc="Source catalog created by characterize task", 

77 name="icSrc", 

78 storageClass="SourceCatalog", 

79 dimensions=("instrument", "visit", "detector"), 

80 ) 

81 

82 astromRefCat = cT.PrerequisiteInput( 

83 doc="Reference catalog to use for astrometry", 

84 name="cal_ref_cat", 

85 storageClass="SimpleCatalog", 

86 dimensions=("skypix",), 

87 deferLoad=True, 

88 multiple=True, 

89 ) 

90 

91 photoRefCat = cT.PrerequisiteInput( 

92 doc="Reference catalog to use for photometric calibration", 

93 name="cal_ref_cat", 

94 storageClass="SimpleCatalog", 

95 dimensions=("skypix",), 

96 deferLoad=True, 

97 multiple=True 

98 ) 

99 

100 outputExposure = cT.Output( 

101 doc="Exposure after running calibration task", 

102 name="calexp", 

103 storageClass="ExposureF", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 

107 outputCat = cT.Output( 

108 doc="Source catalog produced in calibrate task", 

109 name="src", 

110 storageClass="SourceCatalog", 

111 dimensions=("instrument", "visit", "detector"), 

112 ) 

113 

114 outputBackground = cT.Output( 

115 doc="Background models estimated in calibration task", 

116 name="calexpBackground", 

117 storageClass="Background", 

118 dimensions=("instrument", "visit", "detector"), 

119 ) 

120 

121 matches = cT.Output( 

122 doc="Source/refObj matches from the astrometry solver", 

123 name="srcMatch", 

124 storageClass="Catalog", 

125 dimensions=("instrument", "visit", "detector"), 

126 ) 

127 

128 matchesDenormalized = cT.Output( 

129 doc="Denormalized matches from astrometry solver", 

130 name="srcMatchFull", 

131 storageClass="Catalog", 

132 dimensions=("instrument", "visit", "detector"), 

133 ) 

134 

135 def __init__(self, *, config=None): 

136 super().__init__(config=config) 

137 if config.doWriteMatches is False: 

138 self.outputs.remove("matches") 

139 if config.doWriteMatchesDenormalized is False: 

140 self.outputs.remove("matchesDenormalized") 

141 

142 

143class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

144 """Config for CalibrateTask""" 

145 doWrite = pexConfig.Field( 

146 dtype=bool, 

147 default=True, 

148 doc="Save calibration results?", 

149 ) 

150 doWriteHeavyFootprintsInSources = pexConfig.Field( 

151 dtype=bool, 

152 default=True, 

153 doc="Include HeavyFootprint data in source table? If false then heavy " 

154 "footprints are saved as normal footprints, which saves some space" 

155 ) 

156 doWriteMatches = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Write reference matches (ignored if doWrite false)?", 

160 ) 

161 doWriteMatchesDenormalized = pexConfig.Field( 

162 dtype=bool, 

163 default=False, 

164 doc=("Write reference matches in denormalized format? " 

165 "This format uses more disk space, but is more convenient to " 

166 "read. Ignored if doWriteMatches=False or doWrite=False."), 

167 ) 

168 doAstrometry = pexConfig.Field( 

169 dtype=bool, 

170 default=True, 

171 doc="Perform astrometric calibration?", 

172 ) 

173 astromRefObjLoader = pexConfig.ConfigurableField( 

174 target=LoadIndexedReferenceObjectsTask, 

175 doc="reference object loader for astrometric calibration", 

176 ) 

177 photoRefObjLoader = pexConfig.ConfigurableField( 

178 target=LoadIndexedReferenceObjectsTask, 

179 doc="reference object loader for photometric calibration", 

180 ) 

181 astrometry = pexConfig.ConfigurableField( 

182 target=AstrometryTask, 

183 doc="Perform astrometric calibration to refine the WCS", 

184 ) 

185 requireAstrometry = pexConfig.Field( 

186 dtype=bool, 

187 default=True, 

188 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

189 "false."), 

190 ) 

191 doPhotoCal = pexConfig.Field( 

192 dtype=bool, 

193 default=True, 

194 doc="Perform phometric calibration?", 

195 ) 

196 requirePhotoCal = pexConfig.Field( 

197 dtype=bool, 

198 default=True, 

199 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

200 "false."), 

201 ) 

202 photoCal = pexConfig.ConfigurableField( 

203 target=PhotoCalTask, 

204 doc="Perform photometric calibration", 

205 ) 

206 icSourceFieldsToCopy = pexConfig.ListField( 

207 dtype=str, 

208 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

209 doc=("Fields to copy from the icSource catalog to the output catalog " 

210 "for matching sources Any missing fields will trigger a " 

211 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

212 ) 

213 matchRadiusPix = pexConfig.Field( 

214 dtype=float, 

215 default=3, 

216 doc=("Match radius for matching icSourceCat objects to sourceCat " 

217 "objects (pixels)"), 

218 ) 

219 checkUnitsParseStrict = pexConfig.Field( 

220 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

221 "'warn' or 'silent'"), 

222 dtype=str, 

223 default="raise", 

224 ) 

225 detection = pexConfig.ConfigurableField( 

226 target=SourceDetectionTask, 

227 doc="Detect sources" 

228 ) 

229 doDeblend = pexConfig.Field( 

230 dtype=bool, 

231 default=True, 

232 doc="Run deblender input exposure" 

233 ) 

234 deblend = pexConfig.ConfigurableField( 

235 target=SourceDeblendTask, 

236 doc="Split blended sources into their components" 

237 ) 

238 measurement = pexConfig.ConfigurableField( 

239 target=SingleFrameMeasurementTask, 

240 doc="Measure sources" 

241 ) 

242 doApCorr = pexConfig.Field( 

243 dtype=bool, 

244 default=True, 

245 doc="Run subtask to apply aperture correction" 

246 ) 

247 applyApCorr = pexConfig.ConfigurableField( 

248 target=ApplyApCorrTask, 

249 doc="Subtask to apply aperture corrections" 

250 ) 

251 # If doApCorr is False, and the exposure does not have apcorrections 

252 # already applied, the active plugins in catalogCalculation almost 

253 # certainly should not contain the characterization plugin 

254 catalogCalculation = pexConfig.ConfigurableField( 

255 target=CatalogCalculationTask, 

256 doc="Subtask to run catalogCalculation plugins on catalog" 

257 ) 

258 doInsertFakes = pexConfig.Field( 

259 dtype=bool, 

260 default=False, 

261 doc="Run fake sources injection task" 

262 ) 

263 insertFakes = pexConfig.ConfigurableField( 

264 target=BaseFakeSourcesTask, 

265 doc="Injection of fake sources for testing purposes (must be " 

266 "retargeted)" 

267 ) 

268 doWriteExposure = pexConfig.Field( 

269 dtype=bool, 

270 default=True, 

271 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

272 "normal calexp but as a fakes_calexp." 

273 ) 

274 

275 def setDefaults(self): 

276 super().setDefaults() 

277 self.detection.doTempLocalBackground = False 

278 self.deblend.maxFootprintSize = 2000 

279 

280 def validate(self): 

281 super().validate() 

282 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

283 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 

284 raise ValueError( 

285 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

286 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

287 ) 

288 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

289 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 

290 raise ValueError( 

291 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

292 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

293 ) 

294 

295 

296## \addtogroup LSST_task_documentation 

297## \{ 

298## \page CalibrateTask 

299## \ref CalibrateTask_ "CalibrateTask" 

300## \copybrief CalibrateTask 

301## \} 

302 

303class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

304 r"""!Calibrate an exposure: measure sources and perform astrometric and 

305 photometric calibration 

306 

307 @anchor CalibrateTask_ 

308 

309 @section pipe_tasks_calibrate_Contents Contents 

310 

311 - @ref pipe_tasks_calibrate_Purpose 

312 - @ref pipe_tasks_calibrate_Initialize 

313 - @ref pipe_tasks_calibrate_IO 

314 - @ref pipe_tasks_calibrate_Config 

315 - @ref pipe_tasks_calibrate_Metadata 

316 - @ref pipe_tasks_calibrate_Debug 

317 

318 

319 @section pipe_tasks_calibrate_Purpose Description 

320 

321 Given an exposure with a good PSF model and aperture correction map 

322 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

323 operations: 

324 - Run detection and measurement 

325 - Run astrometry subtask to fit an improved WCS 

326 - Run photoCal subtask to fit the exposure's photometric zero-point 

327 

328 @section pipe_tasks_calibrate_Initialize Task initialisation 

329 

330 @copydoc \_\_init\_\_ 

331 

332 @section pipe_tasks_calibrate_IO Invoking the Task 

333 

334 If you want this task to unpersist inputs or persist outputs, then call 

335 the `runDataRef` method (a wrapper around the `run` method). 

336 

337 If you already have the inputs unpersisted and do not want to persist the 

338 output then it is more direct to call the `run` method: 

339 

340 @section pipe_tasks_calibrate_Config Configuration parameters 

341 

342 See @ref CalibrateConfig 

343 

344 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

345 

346 Exposure metadata 

347 <dl> 

348 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

349 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

350 task 

351 <dt>COLORTERM1 <dd>?? (always 0.0) 

352 <dt>COLORTERM2 <dd>?? (always 0.0) 

353 <dt>COLORTERM3 <dd>?? (always 0.0) 

354 </dl> 

355 

356 @section pipe_tasks_calibrate_Debug Debug variables 

357 

358 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

359 interface supports a flag 

360 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

361 for more about `debug.py`. 

362 

363 CalibrateTask has a debug dictionary containing one key: 

364 <dl> 

365 <dt>calibrate 

366 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

367 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

368 the meaning of the various symbols. 

369 </dl> 

370 

371 For example, put something like: 

372 @code{.py} 

373 import lsstDebug 

374 def DebugInfo(name): 

375 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

376 # call us recursively 

377 if name == "lsst.pipe.tasks.calibrate": 

378 di.display = dict( 

379 calibrate = 1, 

380 ) 

381 

382 return di 

383 

384 lsstDebug.Info = DebugInfo 

385 @endcode 

386 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

387 flag. 

388 

389 Some subtasks may have their own debug variables; see individual Task 

390 documentation. 

391 """ 

392 

393 # Example description used to live here, removed 2-20-2017 as per 

394 # https://jira.lsstcorp.org/browse/DM-9520 

395 

396 ConfigClass = CalibrateConfig 

397 _DefaultName = "calibrate" 

398 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

399 

400 def __init__(self, butler=None, astromRefObjLoader=None, 

401 photoRefObjLoader=None, icSourceSchema=None, 

402 initInputs=None, **kwargs): 

403 """!Construct a CalibrateTask 

404 

405 @param[in] butler The butler is passed to the refObjLoader constructor 

406 in case it is needed. Ignored if the refObjLoader argument 

407 provides a loader directly. 

408 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

409 that supplies an external reference catalog for astrometric 

410 calibration. May be None if the desired loader can be constructed 

411 from the butler argument or all steps requiring a reference catalog 

412 are disabled. 

413 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

414 that supplies an external reference catalog for photometric 

415 calibration. May be None if the desired loader can be constructed 

416 from the butler argument or all steps requiring a reference catalog 

417 are disabled. 

418 @param[in] icSourceSchema schema for icSource catalog, or None. 

419 Schema values specified in config.icSourceFieldsToCopy will be 

420 taken from this schema. If set to None, no values will be 

421 propagated from the icSourceCatalog 

422 @param[in,out] kwargs other keyword arguments for 

423 lsst.pipe.base.CmdLineTask 

424 """ 

425 super().__init__(**kwargs) 

426 

427 if icSourceSchema is None and butler is not None: 

428 # Use butler to read icSourceSchema from disk. 

429 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

430 

431 if icSourceSchema is None and butler is None and initInputs is not None: 

432 icSourceSchema = initInputs['icSourceSchema'].schema 

433 

434 if icSourceSchema is not None: 

435 # use a schema mapper to avoid copying each field separately 

436 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

437 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

438 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

439 

440 # Add fields to copy from an icSource catalog 

441 # and a field to indicate that the source matched a source in that 

442 # catalog. If any fields are missing then raise an exception, but 

443 # first find all missing fields in order to make the error message 

444 # more useful. 

445 self.calibSourceKey = self.schemaMapper.addOutputField( 

446 afwTable.Field["Flag"]("calib_detected", 

447 "Source was detected as an icSource")) 

448 missingFieldNames = [] 

449 for fieldName in self.config.icSourceFieldsToCopy: 

450 try: 

451 schemaItem = icSourceSchema.find(fieldName) 

452 except Exception: 

453 missingFieldNames.append(fieldName) 

454 else: 

455 # field found; if addMapping fails then raise an exception 

456 self.schemaMapper.addMapping(schemaItem.getKey()) 

457 

458 if missingFieldNames: 

459 raise RuntimeError("isSourceCat is missing fields {} " 

460 "specified in icSourceFieldsToCopy" 

461 .format(missingFieldNames)) 

462 

463 # produce a temporary schema to pass to the subtasks; finalize it 

464 # later 

465 self.schema = self.schemaMapper.editOutputSchema() 

466 else: 

467 self.schemaMapper = None 

468 self.schema = afwTable.SourceTable.makeMinimalSchema() 

469 self.makeSubtask('detection', schema=self.schema) 

470 

471 self.algMetadata = dafBase.PropertyList() 

472 

473 # Only create a subtask for fakes if configuration option is set 

474 # N.B. the config for fake object task must be retargeted to a child 

475 # of BaseFakeSourcesTask 

476 if self.config.doInsertFakes: 

477 self.makeSubtask("insertFakes") 

478 

479 if self.config.doDeblend: 

480 self.makeSubtask("deblend", schema=self.schema) 

481 self.makeSubtask('measurement', schema=self.schema, 

482 algMetadata=self.algMetadata) 

483 if self.config.doApCorr: 

484 self.makeSubtask('applyApCorr', schema=self.schema) 

485 self.makeSubtask('catalogCalculation', schema=self.schema) 

486 

487 if self.config.doAstrometry: 

488 if astromRefObjLoader is None and butler is not None: 

489 self.makeSubtask('astromRefObjLoader', butler=butler) 

490 astromRefObjLoader = self.astromRefObjLoader 

491 self.pixelMargin = astromRefObjLoader.config.pixelMargin 

492 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

493 schema=self.schema) 

494 if self.config.doPhotoCal: 

495 if photoRefObjLoader is None and butler is not None: 

496 self.makeSubtask('photoRefObjLoader', butler=butler) 

497 photoRefObjLoader = self.photoRefObjLoader 

498 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

499 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

500 schema=self.schema) 

501 

502 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 

503 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

504 "reference object loaders.") 

505 

506 if self.schemaMapper is not None: 

507 # finalize the schema 

508 self.schema = self.schemaMapper.getOutputSchema() 

509 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

510 

511 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

512 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

513 self.outputSchema = sourceCatSchema 

514 

515 @pipeBase.timeMethod 

516 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

517 doUnpersist=True): 

518 """!Calibrate an exposure, optionally unpersisting inputs and 

519 persisting outputs. 

520 

521 This is a wrapper around the `run` method that unpersists inputs 

522 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

523 

524 @param[in] dataRef butler data reference corresponding to a science 

525 image 

526 @param[in,out] exposure characterized exposure (an 

527 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

528 icExp and icBackground. See `run` method for details of what is 

529 read and written. 

530 @param[in,out] background initial model of background already 

531 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

532 None if no background has been subtracted, though that is unusual 

533 for calibration. A refined background model is output. Ignored if 

534 exposure is None. 

535 @param[in] icSourceCat catalog from which to copy the fields specified 

536 by icSourceKeys, or None; 

537 @param[in] doUnpersist unpersist data: 

538 - if True, exposure, background and icSourceCat are read from 

539 dataRef and those three arguments must all be None; 

540 - if False the exposure must be provided; background and 

541 icSourceCat are optional. True is intended for running as a 

542 command-line task, False for running as a subtask 

543 @return same data as the calibrate method 

544 """ 

545 self.log.info("Processing %s" % (dataRef.dataId)) 

546 

547 if doUnpersist: 

548 if any(item is not None for item in (exposure, background, 

549 icSourceCat)): 

550 raise RuntimeError("doUnpersist true; exposure, background " 

551 "and icSourceCat must all be None") 

552 exposure = dataRef.get("icExp", immediate=True) 

553 background = dataRef.get("icExpBackground", immediate=True) 

554 icSourceCat = dataRef.get("icSrc", immediate=True) 

555 elif exposure is None: 

556 raise RuntimeError("doUnpersist false; exposure must be provided") 

557 

558 exposureIdInfo = dataRef.get("expIdInfo") 

559 

560 calRes = self.run( 

561 exposure=exposure, 

562 exposureIdInfo=exposureIdInfo, 

563 background=background, 

564 icSourceCat=icSourceCat, 

565 ) 

566 

567 if self.config.doWrite: 

568 self.writeOutputs( 

569 dataRef=dataRef, 

570 exposure=calRes.exposure, 

571 background=calRes.background, 

572 sourceCat=calRes.sourceCat, 

573 astromMatches=calRes.astromMatches, 

574 matchMeta=calRes.matchMeta, 

575 ) 

576 

577 return calRes 

578 

579 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

580 inputs = butlerQC.get(inputRefs) 

581 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

582 returnMaxBits=True) 

583 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

584 

585 if self.config.doAstrometry: 

586 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

587 for ref in inputRefs.astromRefCat], 

588 refCats=inputs.pop('astromRefCat'), 

589 config=self.config.astromRefObjLoader, log=self.log) 

590 self.pixelMargin = refObjLoader.config.pixelMargin 

591 self.astrometry.setRefObjLoader(refObjLoader) 

592 

593 if self.config.doPhotoCal: 

594 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

595 for ref in inputRefs.photoRefCat], 

596 refCats=inputs.pop('photoRefCat'), 

597 config=self.config.photoRefObjLoader, 

598 log=self.log) 

599 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

600 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

601 

602 outputs = self.run(**inputs) 

603 

604 if self.config.doWriteMatches: 

605 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

606 normalizedMatches.table.setMetadata(outputs.matchMeta) 

607 if self.config.doWriteMatchesDenormalized: 

608 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

609 outputs.matchesDenormalized = denormMatches 

610 outputs.matches = normalizedMatches 

611 butlerQC.put(outputs, outputRefs) 

612 

613 def run(self, exposure, exposureIdInfo=None, background=None, 

614 icSourceCat=None): 

615 """!Calibrate an exposure (science image or coadd) 

616 

617 @param[in,out] exposure exposure to calibrate (an 

618 lsst.afw.image.ExposureF or similar); 

619 in: 

620 - MaskedImage 

621 - Psf 

622 out: 

623 - MaskedImage has background subtracted 

624 - Wcs is replaced 

625 - PhotoCalib is replaced 

626 @param[in] exposureIdInfo ID info for exposure (an 

627 lsst.obs.base.ExposureIdInfo) If not provided, returned 

628 SourceCatalog IDs will not be globally unique. 

629 @param[in,out] background background model already subtracted from 

630 exposure (an lsst.afw.math.BackgroundList). May be None if no 

631 background has been subtracted, though that is unusual for 

632 calibration. A refined background model is output. 

633 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

634 from which we can copy some fields. 

635 

636 @return pipe_base Struct containing these fields: 

637 - exposure calibrate science exposure with refined WCS and PhotoCalib 

638 - background model of background subtracted from exposure (an 

639 lsst.afw.math.BackgroundList) 

640 - sourceCat catalog of measured sources 

641 - astromMatches list of source/refObj matches from the astrometry 

642 solver 

643 """ 

644 # detect, deblend and measure sources 

645 if exposureIdInfo is None: 

646 exposureIdInfo = ExposureIdInfo() 

647 

648 if background is None: 

649 background = BackgroundList() 

650 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, 

651 exposureIdInfo.unusedBits) 

652 table = SourceTable.make(self.schema, sourceIdFactory) 

653 table.setMetadata(self.algMetadata) 

654 

655 detRes = self.detection.run(table=table, exposure=exposure, 

656 doSmooth=True) 

657 sourceCat = detRes.sources 

658 if detRes.fpSets.background: 

659 for bg in detRes.fpSets.background: 

660 background.append(bg) 

661 if self.config.doDeblend: 

662 self.deblend.run(exposure=exposure, sources=sourceCat) 

663 self.measurement.run( 

664 measCat=sourceCat, 

665 exposure=exposure, 

666 exposureId=exposureIdInfo.expId 

667 ) 

668 if self.config.doApCorr: 

669 self.applyApCorr.run( 

670 catalog=sourceCat, 

671 apCorrMap=exposure.getInfo().getApCorrMap() 

672 ) 

673 self.catalogCalculation.run(sourceCat) 

674 

675 if icSourceCat is not None and \ 

676 len(self.config.icSourceFieldsToCopy) > 0: 

677 self.copyIcSourceFields(icSourceCat=icSourceCat, 

678 sourceCat=sourceCat) 

679 

680 # TODO DM-11568: this contiguous check-and-copy could go away if we 

681 # reserve enough space during SourceDetection and/or SourceDeblend. 

682 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

683 # contiguity now, so views are preserved from here on. 

684 if not sourceCat.isContiguous(): 

685 sourceCat = sourceCat.copy(deep=True) 

686 

687 # perform astrometry calibration: 

688 # fit an improved WCS and update the exposure's WCS in place 

689 astromMatches = None 

690 matchMeta = None 

691 if self.config.doAstrometry: 

692 try: 

693 astromRes = self.astrometry.run( 

694 exposure=exposure, 

695 sourceCat=sourceCat, 

696 ) 

697 astromMatches = astromRes.matches 

698 matchMeta = astromRes.matchMeta 

699 except Exception as e: 

700 if self.config.requireAstrometry: 

701 raise 

702 self.log.warn("Unable to perform astrometric calibration " 

703 "(%s): attempting to proceed" % e) 

704 

705 # compute photometric calibration 

706 if self.config.doPhotoCal: 

707 try: 

708 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

709 exposure.setPhotoCalib(photoRes.photoCalib) 

710 # TODO: reword this to phrase it in terms of the calibration factor? 

711 self.log.info("Photometric zero-point: %f" % 

712 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

713 self.setMetadata(exposure=exposure, photoRes=photoRes) 

714 except Exception as e: 

715 if self.config.requirePhotoCal: 

716 raise 

717 self.log.warn("Unable to perform photometric calibration " 

718 "(%s): attempting to proceed" % e) 

719 self.setMetadata(exposure=exposure, photoRes=None) 

720 

721 if self.config.doInsertFakes: 

722 self.insertFakes.run(exposure, background=background) 

723 

724 table = SourceTable.make(self.schema, sourceIdFactory) 

725 table.setMetadata(self.algMetadata) 

726 

727 detRes = self.detection.run(table=table, exposure=exposure, 

728 doSmooth=True) 

729 sourceCat = detRes.sources 

730 if detRes.fpSets.background: 

731 for bg in detRes.fpSets.background: 

732 background.append(bg) 

733 if self.config.doDeblend: 

734 self.deblend.run(exposure=exposure, sources=sourceCat) 

735 self.measurement.run( 

736 measCat=sourceCat, 

737 exposure=exposure, 

738 exposureId=exposureIdInfo.expId 

739 ) 

740 if self.config.doApCorr: 

741 self.applyApCorr.run( 

742 catalog=sourceCat, 

743 apCorrMap=exposure.getInfo().getApCorrMap() 

744 ) 

745 self.catalogCalculation.run(sourceCat) 

746 

747 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 

748 self.copyIcSourceFields(icSourceCat=icSourceCat, 

749 sourceCat=sourceCat) 

750 

751 frame = getDebugFrame(self._display, "calibrate") 

752 if frame: 

753 displayAstrometry( 

754 sourceCat=sourceCat, 

755 exposure=exposure, 

756 matches=astromMatches, 

757 frame=frame, 

758 pause=False, 

759 ) 

760 

761 return pipeBase.Struct( 

762 exposure=exposure, 

763 background=background, 

764 sourceCat=sourceCat, 

765 astromMatches=astromMatches, 

766 matchMeta=matchMeta, 

767 # These are duplicate entries with different names for use with 

768 # gen3 middleware 

769 outputExposure=exposure, 

770 outputCat=sourceCat, 

771 outputBackground=background, 

772 ) 

773 

774 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

775 astromMatches, matchMeta): 

776 """Write output data to the output repository 

777 

778 @param[in] dataRef butler data reference corresponding to a science 

779 image 

780 @param[in] exposure exposure to write 

781 @param[in] background background model for exposure 

782 @param[in] sourceCat catalog of measured sources 

783 @param[in] astromMatches list of source/refObj matches from the 

784 astrometry solver 

785 """ 

786 dataRef.put(sourceCat, "src") 

787 if self.config.doWriteMatches and astromMatches is not None: 

788 normalizedMatches = afwTable.packMatches(astromMatches) 

789 normalizedMatches.table.setMetadata(matchMeta) 

790 dataRef.put(normalizedMatches, "srcMatch") 

791 if self.config.doWriteMatchesDenormalized: 

792 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

793 dataRef.put(denormMatches, "srcMatchFull") 

794 if self.config.doWriteExposure: 

795 dataRef.put(exposure, "calexp") 

796 dataRef.put(background, "calexpBackground") 

797 

798 def getSchemaCatalogs(self): 

799 """Return a dict of empty catalogs for each catalog dataset produced 

800 by this task. 

801 """ 

802 sourceCat = afwTable.SourceCatalog(self.schema) 

803 sourceCat.getTable().setMetadata(self.algMetadata) 

804 return {"src": sourceCat} 

805 

806 def setMetadata(self, exposure, photoRes=None): 

807 """!Set task and exposure metadata 

808 

809 Logs a warning and continues if needed data is missing. 

810 

811 @param[in,out] exposure exposure whose metadata is to be set 

812 @param[in] photoRes results of running photoCal; if None then it was 

813 not run 

814 """ 

815 if photoRes is None: 

816 return 

817 

818 metadata = exposure.getMetadata() 

819 

820 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

821 try: 

822 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

823 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

824 except Exception: 

825 self.log.warn("Could not set normalized MAGZERO in header: no " 

826 "exposure time") 

827 magZero = math.nan 

828 

829 try: 

830 metadata.set('MAGZERO', magZero) 

831 metadata.set('MAGZERO_RMS', photoRes.sigma) 

832 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

833 metadata.set('COLORTERM1', 0.0) 

834 metadata.set('COLORTERM2', 0.0) 

835 metadata.set('COLORTERM3', 0.0) 

836 except Exception as e: 

837 self.log.warn("Could not set exposure metadata: %s" % (e,)) 

838 

839 def copyIcSourceFields(self, icSourceCat, sourceCat): 

840 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

841 

842 @param[in] icSourceCat catalog from which to copy fields 

843 @param[in,out] sourceCat catalog to which to copy fields 

844 

845 The fields copied are those specified by `config.icSourceFieldsToCopy` 

846 that actually exist in the schema. This was set up by the constructor 

847 using self.schemaMapper. 

848 """ 

849 if self.schemaMapper is None: 

850 raise RuntimeError("To copy icSource fields you must specify " 

851 "icSourceSchema nd icSourceKeys when " 

852 "constructing this task") 

853 if icSourceCat is None or sourceCat is None: 

854 raise RuntimeError("icSourceCat and sourceCat must both be " 

855 "specified") 

856 if len(self.config.icSourceFieldsToCopy) == 0: 

857 self.log.warn("copyIcSourceFields doing nothing because " 

858 "icSourceFieldsToCopy is empty") 

859 return 

860 

861 mc = afwTable.MatchControl() 

862 mc.findOnlyClosest = False # return all matched objects 

863 matches = afwTable.matchXy(icSourceCat, sourceCat, 

864 self.config.matchRadiusPix, mc) 

865 if self.config.doDeblend: 

866 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

867 # if deblended, keep children 

868 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

869 

870 # Because we had to allow multiple matches to handle parents, we now 

871 # need to prune to the best matches 

872 # closest matches as a dict of icSourceCat source ID: 

873 # (icSourceCat source, sourceCat source, distance in pixels) 

874 bestMatches = {} 

875 for m0, m1, d in matches: 

876 id0 = m0.getId() 

877 match = bestMatches.get(id0) 

878 if match is None or d <= match[2]: 

879 bestMatches[id0] = (m0, m1, d) 

880 matches = list(bestMatches.values()) 

881 

882 # Check that no sourceCat sources are listed twice (we already know 

883 # that each match has a unique icSourceCat source ID, due to using 

884 # that ID as the key in bestMatches) 

885 numMatches = len(matches) 

886 numUniqueSources = len(set(m[1].getId() for m in matches)) 

887 if numUniqueSources != numMatches: 

888 self.log.warn("{} icSourceCat sources matched only {} sourceCat " 

889 "sources".format(numMatches, numUniqueSources)) 

890 

891 self.log.info("Copying flags from icSourceCat to sourceCat for " 

892 "%s sources" % (numMatches,)) 

893 

894 # For each match: set the calibSourceKey flag and copy the desired 

895 # fields 

896 for icSrc, src, d in matches: 

897 src.setFlag(self.calibSourceKey, True) 

898 # src.assign copies the footprint from icSrc, which we don't want 

899 # (DM-407) 

900 # so set icSrc's footprint to src's footprint before src.assign, 

901 # then restore it 

902 icSrcFootprint = icSrc.getFootprint() 

903 try: 

904 icSrc.setFootprint(src.getFootprint()) 

905 src.assign(icSrc, self.schemaMapper) 

906 finally: 

907 icSrc.setFootprint(icSrcFootprint)