Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import math 

23 

24from lsstDebug import getDebugFrame 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.table as afwTable 

29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches 

30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask 

31from lsst.obs.base import ExposureIdInfo 

32import lsst.daf.base as dafBase 

33from lsst.afw.math import BackgroundList 

34from lsst.afw.table import IdFactory, SourceTable 

35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader 

36from lsst.meas.base import (SingleFrameMeasurementTask, 

37 ApplyApCorrTask, 

38 CatalogCalculationTask) 

39from lsst.meas.deblender import SourceDeblendTask 

40from .fakes import BaseFakeSourcesTask 

41from .photoCal import PhotoCalTask 

42 

43__all__ = ["CalibrateConfig", "CalibrateTask"] 

44 

45 

46class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), 

47 defaultTemplates={}): 

48 

49 icSourceSchema = cT.InitInput( 

50 doc="Schema produced by characterize image task, used to initialize this task", 

51 name="icSrc_schema", 

52 storageClass="SourceCatalog", 

53 ) 

54 

55 outputSchema = cT.InitOutput( 

56 doc="Schema after CalibrateTask has been initialized", 

57 name="src_schema", 

58 storageClass="SourceCatalog", 

59 ) 

60 

61 exposure = cT.Input( 

62 doc="Input image to calibrate", 

63 name="icExp", 

64 storageClass="ExposureF", 

65 dimensions=("instrument", "visit", "detector"), 

66 ) 

67 

68 background = cT.Input( 

69 doc="Backgrounds determined by characterize task", 

70 name="icExpBackground", 

71 storageClass="Background", 

72 dimensions=("instrument", "visit", "detector"), 

73 ) 

74 

75 icSourceCat = cT.Input( 

76 doc="Source catalog created by characterize task", 

77 name="icSrc", 

78 storageClass="SourceCatalog", 

79 dimensions=("instrument", "visit", "detector"), 

80 ) 

81 

82 astromRefCat = cT.PrerequisiteInput( 

83 doc="Reference catalog to use for astrometry", 

84 name="cal_ref_cat", 

85 storageClass="SimpleCatalog", 

86 dimensions=("skypix",), 

87 deferLoad=True, 

88 multiple=True, 

89 ) 

90 

91 photoRefCat = cT.PrerequisiteInput( 

92 doc="Reference catalog to use for photometric calibration", 

93 name="cal_ref_cat", 

94 storageClass="SimpleCatalog", 

95 dimensions=("skypix",), 

96 deferLoad=True, 

97 multiple=True 

98 ) 

99 

100 outputExposure = cT.Output( 

101 doc="Exposure after running calibration task", 

102 name="calexp", 

103 storageClass="ExposureF", 

104 dimensions=("instrument", "visit", "detector"), 

105 ) 

106 

107 outputCat = cT.Output( 

108 doc="Source catalog produced in calibrate task", 

109 name="src", 

110 storageClass="SourceCatalog", 

111 dimensions=("instrument", "visit", "detector"), 

112 ) 

113 

114 outputBackground = cT.Output( 

115 doc="Background models estimated in calibration task", 

116 name="calexpBackground", 

117 storageClass="Background", 

118 dimensions=("instrument", "visit", "detector"), 

119 ) 

120 

121 matches = cT.Output( 

122 doc="Source/refObj matches from the astrometry solver", 

123 name="srcMatch", 

124 storageClass="Catalog", 

125 dimensions=("instrument", "visit", "detector"), 

126 ) 

127 

128 matchesDenormalized = cT.Output( 

129 doc="Denormalized matches from astrometry solver", 

130 name="srcMatchFull", 

131 storageClass="Catalog", 

132 dimensions=("instrument", "visit", "detector"), 

133 ) 

134 

135 def __init__(self, *, config=None): 

136 super().__init__(config=config) 

137 

138 if config.doAstrometry is False: 

139 self.prerequisiteInputs.remove("astromRefCat") 

140 if config.doPhotoCal is False: 

141 self.prerequisiteInputs.remove("photoRefCat") 

142 

143 if config.doWriteMatches is False or config.doAstrometry is False: 

144 self.outputs.remove("matches") 

145 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: 

146 self.outputs.remove("matchesDenormalized") 

147 

148 

149class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): 

150 """Config for CalibrateTask""" 

151 doWrite = pexConfig.Field( 

152 dtype=bool, 

153 default=True, 

154 doc="Save calibration results?", 

155 ) 

156 doWriteHeavyFootprintsInSources = pexConfig.Field( 

157 dtype=bool, 

158 default=True, 

159 doc="Include HeavyFootprint data in source table? If false then heavy " 

160 "footprints are saved as normal footprints, which saves some space" 

161 ) 

162 doWriteMatches = pexConfig.Field( 

163 dtype=bool, 

164 default=True, 

165 doc="Write reference matches (ignored if doWrite or doAstrometry false)?", 

166 ) 

167 doWriteMatchesDenormalized = pexConfig.Field( 

168 dtype=bool, 

169 default=False, 

170 doc=("Write reference matches in denormalized format? " 

171 "This format uses more disk space, but is more convenient to " 

172 "read. Ignored if doWriteMatches=False or doWrite=False."), 

173 ) 

174 doAstrometry = pexConfig.Field( 

175 dtype=bool, 

176 default=True, 

177 doc="Perform astrometric calibration?", 

178 ) 

179 astromRefObjLoader = pexConfig.ConfigurableField( 

180 target=LoadIndexedReferenceObjectsTask, 

181 doc="reference object loader for astrometric calibration", 

182 ) 

183 photoRefObjLoader = pexConfig.ConfigurableField( 

184 target=LoadIndexedReferenceObjectsTask, 

185 doc="reference object loader for photometric calibration", 

186 ) 

187 astrometry = pexConfig.ConfigurableField( 

188 target=AstrometryTask, 

189 doc="Perform astrometric calibration to refine the WCS", 

190 ) 

191 requireAstrometry = pexConfig.Field( 

192 dtype=bool, 

193 default=True, 

194 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry " 

195 "false."), 

196 ) 

197 doPhotoCal = pexConfig.Field( 

198 dtype=bool, 

199 default=True, 

200 doc="Perform phometric calibration?", 

201 ) 

202 requirePhotoCal = pexConfig.Field( 

203 dtype=bool, 

204 default=True, 

205 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal " 

206 "false."), 

207 ) 

208 photoCal = pexConfig.ConfigurableField( 

209 target=PhotoCalTask, 

210 doc="Perform photometric calibration", 

211 ) 

212 icSourceFieldsToCopy = pexConfig.ListField( 

213 dtype=str, 

214 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"), 

215 doc=("Fields to copy from the icSource catalog to the output catalog " 

216 "for matching sources Any missing fields will trigger a " 

217 "RuntimeError exception. Ignored if icSourceCat is not provided.") 

218 ) 

219 matchRadiusPix = pexConfig.Field( 

220 dtype=float, 

221 default=3, 

222 doc=("Match radius for matching icSourceCat objects to sourceCat " 

223 "objects (pixels)"), 

224 ) 

225 checkUnitsParseStrict = pexConfig.Field( 

226 doc=("Strictness of Astropy unit compatibility check, can be 'raise', " 

227 "'warn' or 'silent'"), 

228 dtype=str, 

229 default="raise", 

230 ) 

231 detection = pexConfig.ConfigurableField( 

232 target=SourceDetectionTask, 

233 doc="Detect sources" 

234 ) 

235 doDeblend = pexConfig.Field( 

236 dtype=bool, 

237 default=True, 

238 doc="Run deblender input exposure" 

239 ) 

240 deblend = pexConfig.ConfigurableField( 

241 target=SourceDeblendTask, 

242 doc="Split blended sources into their components" 

243 ) 

244 doSkySources = pexConfig.Field( 

245 dtype=bool, 

246 default=True, 

247 doc="Generate sky sources?", 

248 ) 

249 skySources = pexConfig.ConfigurableField( 

250 target=SkyObjectsTask, 

251 doc="Generate sky sources", 

252 ) 

253 measurement = pexConfig.ConfigurableField( 

254 target=SingleFrameMeasurementTask, 

255 doc="Measure sources" 

256 ) 

257 doApCorr = pexConfig.Field( 

258 dtype=bool, 

259 default=True, 

260 doc="Run subtask to apply aperture correction" 

261 ) 

262 applyApCorr = pexConfig.ConfigurableField( 

263 target=ApplyApCorrTask, 

264 doc="Subtask to apply aperture corrections" 

265 ) 

266 # If doApCorr is False, and the exposure does not have apcorrections 

267 # already applied, the active plugins in catalogCalculation almost 

268 # certainly should not contain the characterization plugin 

269 catalogCalculation = pexConfig.ConfigurableField( 

270 target=CatalogCalculationTask, 

271 doc="Subtask to run catalogCalculation plugins on catalog" 

272 ) 

273 doInsertFakes = pexConfig.Field( 

274 dtype=bool, 

275 default=False, 

276 doc="Run fake sources injection task" 

277 ) 

278 insertFakes = pexConfig.ConfigurableField( 

279 target=BaseFakeSourcesTask, 

280 doc="Injection of fake sources for testing purposes (must be " 

281 "retargeted)" 

282 ) 

283 doWriteExposure = pexConfig.Field( 

284 dtype=bool, 

285 default=True, 

286 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a " 

287 "normal calexp but as a fakes_calexp." 

288 ) 

289 

290 def setDefaults(self): 

291 super().setDefaults() 

292 self.detection.doTempLocalBackground = False 

293 self.deblend.maxFootprintSize = 2000 

294 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"] 

295 

296 def validate(self): 

297 super().validate() 

298 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None) 

299 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat: 299 ↛ 300line 299 didn't jump to line 300, because the condition on line 299 was never true

300 raise ValueError( 

301 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference " 

302 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

303 ) 

304 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None) 

305 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat: 305 ↛ 306line 305 didn't jump to line 306, because the condition on line 305 was never true

306 raise ValueError( 

307 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference " 

308 f"catalogs are different. These options must be kept in sync until Gen2 is retired." 

309 ) 

310 

311 

312## \addtogroup LSST_task_documentation 

313## \{ 

314## \page CalibrateTask 

315## \ref CalibrateTask_ "CalibrateTask" 

316## \copybrief CalibrateTask 

317## \} 

318 

319class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

320 r"""!Calibrate an exposure: measure sources and perform astrometric and 

321 photometric calibration 

322 

323 @anchor CalibrateTask_ 

324 

325 @section pipe_tasks_calibrate_Contents Contents 

326 

327 - @ref pipe_tasks_calibrate_Purpose 

328 - @ref pipe_tasks_calibrate_Initialize 

329 - @ref pipe_tasks_calibrate_IO 

330 - @ref pipe_tasks_calibrate_Config 

331 - @ref pipe_tasks_calibrate_Metadata 

332 - @ref pipe_tasks_calibrate_Debug 

333 

334 

335 @section pipe_tasks_calibrate_Purpose Description 

336 

337 Given an exposure with a good PSF model and aperture correction map 

338 (e.g. as provided by @ref CharacterizeImageTask), perform the following 

339 operations: 

340 - Run detection and measurement 

341 - Run astrometry subtask to fit an improved WCS 

342 - Run photoCal subtask to fit the exposure's photometric zero-point 

343 

344 @section pipe_tasks_calibrate_Initialize Task initialisation 

345 

346 @copydoc \_\_init\_\_ 

347 

348 @section pipe_tasks_calibrate_IO Invoking the Task 

349 

350 If you want this task to unpersist inputs or persist outputs, then call 

351 the `runDataRef` method (a wrapper around the `run` method). 

352 

353 If you already have the inputs unpersisted and do not want to persist the 

354 output then it is more direct to call the `run` method: 

355 

356 @section pipe_tasks_calibrate_Config Configuration parameters 

357 

358 See @ref CalibrateConfig 

359 

360 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 

361 

362 Exposure metadata 

363 <dl> 

364 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 

365 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 

366 task 

367 <dt>COLORTERM1 <dd>?? (always 0.0) 

368 <dt>COLORTERM2 <dd>?? (always 0.0) 

369 <dt>COLORTERM3 <dd>?? (always 0.0) 

370 </dl> 

371 

372 @section pipe_tasks_calibrate_Debug Debug variables 

373 

374 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 

375 interface supports a flag 

376 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 

377 for more about `debug.py`. 

378 

379 CalibrateTask has a debug dictionary containing one key: 

380 <dl> 

381 <dt>calibrate 

382 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 

383 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 

384 the meaning of the various symbols. 

385 </dl> 

386 

387 For example, put something like: 

388 @code{.py} 

389 import lsstDebug 

390 def DebugInfo(name): 

391 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 

392 # call us recursively 

393 if name == "lsst.pipe.tasks.calibrate": 

394 di.display = dict( 

395 calibrate = 1, 

396 ) 

397 

398 return di 

399 

400 lsstDebug.Info = DebugInfo 

401 @endcode 

402 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 

403 flag. 

404 

405 Some subtasks may have their own debug variables; see individual Task 

406 documentation. 

407 """ 

408 

409 # Example description used to live here, removed 2-20-2017 as per 

410 # https://jira.lsstcorp.org/browse/DM-9520 

411 

412 ConfigClass = CalibrateConfig 

413 _DefaultName = "calibrate" 

414 RunnerClass = pipeBase.ButlerInitializedTaskRunner 

415 

416 def __init__(self, butler=None, astromRefObjLoader=None, 

417 photoRefObjLoader=None, icSourceSchema=None, 

418 initInputs=None, **kwargs): 

419 """!Construct a CalibrateTask 

420 

421 @param[in] butler The butler is passed to the refObjLoader constructor 

422 in case it is needed. Ignored if the refObjLoader argument 

423 provides a loader directly. 

424 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 

425 that supplies an external reference catalog for astrometric 

426 calibration. May be None if the desired loader can be constructed 

427 from the butler argument or all steps requiring a reference catalog 

428 are disabled. 

429 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 

430 that supplies an external reference catalog for photometric 

431 calibration. May be None if the desired loader can be constructed 

432 from the butler argument or all steps requiring a reference catalog 

433 are disabled. 

434 @param[in] icSourceSchema schema for icSource catalog, or None. 

435 Schema values specified in config.icSourceFieldsToCopy will be 

436 taken from this schema. If set to None, no values will be 

437 propagated from the icSourceCatalog 

438 @param[in,out] kwargs other keyword arguments for 

439 lsst.pipe.base.CmdLineTask 

440 """ 

441 super().__init__(**kwargs) 

442 

443 if icSourceSchema is None and butler is not None: 

444 # Use butler to read icSourceSchema from disk. 

445 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema 

446 

447 if icSourceSchema is None and butler is None and initInputs is not None: 447 ↛ 448line 447 didn't jump to line 448, because the condition on line 447 was never true

448 icSourceSchema = initInputs['icSourceSchema'].schema 

449 

450 if icSourceSchema is not None: 

451 # use a schema mapper to avoid copying each field separately 

452 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema) 

453 minimumSchema = afwTable.SourceTable.makeMinimalSchema() 

454 self.schemaMapper.addMinimalSchema(minimumSchema, False) 

455 

456 # Add fields to copy from an icSource catalog 

457 # and a field to indicate that the source matched a source in that 

458 # catalog. If any fields are missing then raise an exception, but 

459 # first find all missing fields in order to make the error message 

460 # more useful. 

461 self.calibSourceKey = self.schemaMapper.addOutputField( 

462 afwTable.Field["Flag"]("calib_detected", 

463 "Source was detected as an icSource")) 

464 missingFieldNames = [] 

465 for fieldName in self.config.icSourceFieldsToCopy: 

466 try: 

467 schemaItem = icSourceSchema.find(fieldName) 

468 except Exception: 

469 missingFieldNames.append(fieldName) 

470 else: 

471 # field found; if addMapping fails then raise an exception 

472 self.schemaMapper.addMapping(schemaItem.getKey()) 

473 

474 if missingFieldNames: 474 ↛ 475line 474 didn't jump to line 475, because the condition on line 474 was never true

475 raise RuntimeError("isSourceCat is missing fields {} " 

476 "specified in icSourceFieldsToCopy" 

477 .format(missingFieldNames)) 

478 

479 # produce a temporary schema to pass to the subtasks; finalize it 

480 # later 

481 self.schema = self.schemaMapper.editOutputSchema() 

482 else: 

483 self.schemaMapper = None 

484 self.schema = afwTable.SourceTable.makeMinimalSchema() 

485 self.makeSubtask('detection', schema=self.schema) 

486 

487 self.algMetadata = dafBase.PropertyList() 

488 

489 # Only create a subtask for fakes if configuration option is set 

490 # N.B. the config for fake object task must be retargeted to a child 

491 # of BaseFakeSourcesTask 

492 if self.config.doInsertFakes: 

493 self.makeSubtask("insertFakes") 

494 

495 if self.config.doDeblend: 495 ↛ 497line 495 didn't jump to line 497, because the condition on line 495 was never false

496 self.makeSubtask("deblend", schema=self.schema) 

497 if self.config.doSkySources: 

498 self.makeSubtask("skySources") 

499 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.") 

500 self.makeSubtask('measurement', schema=self.schema, 

501 algMetadata=self.algMetadata) 

502 if self.config.doApCorr: 

503 self.makeSubtask('applyApCorr', schema=self.schema) 

504 self.makeSubtask('catalogCalculation', schema=self.schema) 

505 

506 if self.config.doAstrometry: 506 ↛ 507line 506 didn't jump to line 507, because the condition on line 506 was never true

507 if astromRefObjLoader is None and butler is not None: 

508 self.makeSubtask('astromRefObjLoader', butler=butler) 

509 astromRefObjLoader = self.astromRefObjLoader 

510 self.pixelMargin = astromRefObjLoader.config.pixelMargin 

511 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader, 

512 schema=self.schema) 

513 if self.config.doPhotoCal: 513 ↛ 514line 513 didn't jump to line 514, because the condition on line 513 was never true

514 if photoRefObjLoader is None and butler is not None: 

515 self.makeSubtask('photoRefObjLoader', butler=butler) 

516 photoRefObjLoader = self.photoRefObjLoader 

517 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

518 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader, 

519 schema=self.schema) 

520 

521 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): 521 ↛ 522line 521 didn't jump to line 522, because the condition on line 521 was never true

522 raise RuntimeError("PipelineTask form of this task should not be initialized with " 

523 "reference object loaders.") 

524 

525 if self.schemaMapper is not None: 

526 # finalize the schema 

527 self.schema = self.schemaMapper.getOutputSchema() 

528 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) 

529 

530 sourceCatSchema = afwTable.SourceCatalog(self.schema) 

531 sourceCatSchema.getTable().setMetadata(self.algMetadata) 

532 self.outputSchema = sourceCatSchema 

533 

534 @pipeBase.timeMethod 

535 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, 

536 doUnpersist=True): 

537 """!Calibrate an exposure, optionally unpersisting inputs and 

538 persisting outputs. 

539 

540 This is a wrapper around the `run` method that unpersists inputs 

541 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 

542 

543 @param[in] dataRef butler data reference corresponding to a science 

544 image 

545 @param[in,out] exposure characterized exposure (an 

546 lsst.afw.image.ExposureF or similar), or None to unpersist existing 

547 icExp and icBackground. See `run` method for details of what is 

548 read and written. 

549 @param[in,out] background initial model of background already 

550 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 

551 None if no background has been subtracted, though that is unusual 

552 for calibration. A refined background model is output. Ignored if 

553 exposure is None. 

554 @param[in] icSourceCat catalog from which to copy the fields specified 

555 by icSourceKeys, or None; 

556 @param[in] doUnpersist unpersist data: 

557 - if True, exposure, background and icSourceCat are read from 

558 dataRef and those three arguments must all be None; 

559 - if False the exposure must be provided; background and 

560 icSourceCat are optional. True is intended for running as a 

561 command-line task, False for running as a subtask 

562 @return same data as the calibrate method 

563 """ 

564 self.log.info("Processing %s" % (dataRef.dataId)) 

565 

566 if doUnpersist: 

567 if any(item is not None for item in (exposure, background, 567 ↛ 569line 567 didn't jump to line 569, because the condition on line 567 was never true

568 icSourceCat)): 

569 raise RuntimeError("doUnpersist true; exposure, background " 

570 "and icSourceCat must all be None") 

571 exposure = dataRef.get("icExp", immediate=True) 

572 background = dataRef.get("icExpBackground", immediate=True) 

573 icSourceCat = dataRef.get("icSrc", immediate=True) 

574 elif exposure is None: 574 ↛ 575line 574 didn't jump to line 575, because the condition on line 574 was never true

575 raise RuntimeError("doUnpersist false; exposure must be provided") 

576 

577 exposureIdInfo = dataRef.get("expIdInfo") 

578 

579 calRes = self.run( 

580 exposure=exposure, 

581 exposureIdInfo=exposureIdInfo, 

582 background=background, 

583 icSourceCat=icSourceCat, 

584 ) 

585 

586 if self.config.doWrite: 586 ↛ 596line 586 didn't jump to line 596, because the condition on line 586 was never false

587 self.writeOutputs( 

588 dataRef=dataRef, 

589 exposure=calRes.exposure, 

590 background=calRes.background, 

591 sourceCat=calRes.sourceCat, 

592 astromMatches=calRes.astromMatches, 

593 matchMeta=calRes.matchMeta, 

594 ) 

595 

596 return calRes 

597 

598 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

599 inputs = butlerQC.get(inputRefs) 

600 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

601 returnMaxBits=True) 

602 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits) 

603 

604 if self.config.doAstrometry: 

605 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

606 for ref in inputRefs.astromRefCat], 

607 refCats=inputs.pop('astromRefCat'), 

608 config=self.config.astromRefObjLoader, log=self.log) 

609 self.pixelMargin = refObjLoader.config.pixelMargin 

610 self.astrometry.setRefObjLoader(refObjLoader) 

611 

612 if self.config.doPhotoCal: 

613 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

614 for ref in inputRefs.photoRefCat], 

615 refCats=inputs.pop('photoRefCat'), 

616 config=self.config.photoRefObjLoader, 

617 log=self.log) 

618 self.pixelMargin = photoRefObjLoader.config.pixelMargin 

619 self.photoCal.match.setRefObjLoader(photoRefObjLoader) 

620 

621 outputs = self.run(**inputs) 

622 

623 if self.config.doWriteMatches and self.config.doAstrometry: 

624 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

625 normalizedMatches.table.setMetadata(outputs.matchMeta) 

626 if self.config.doWriteMatchesDenormalized: 

627 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta) 

628 outputs.matchesDenormalized = denormMatches 

629 outputs.matches = normalizedMatches 

630 butlerQC.put(outputs, outputRefs) 

631 

632 def run(self, exposure, exposureIdInfo=None, background=None, 

633 icSourceCat=None): 

634 """!Calibrate an exposure (science image or coadd) 

635 

636 @param[in,out] exposure exposure to calibrate (an 

637 lsst.afw.image.ExposureF or similar); 

638 in: 

639 - MaskedImage 

640 - Psf 

641 out: 

642 - MaskedImage has background subtracted 

643 - Wcs is replaced 

644 - PhotoCalib is replaced 

645 @param[in] exposureIdInfo ID info for exposure (an 

646 lsst.obs.base.ExposureIdInfo) If not provided, returned 

647 SourceCatalog IDs will not be globally unique. 

648 @param[in,out] background background model already subtracted from 

649 exposure (an lsst.afw.math.BackgroundList). May be None if no 

650 background has been subtracted, though that is unusual for 

651 calibration. A refined background model is output. 

652 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 

653 from which we can copy some fields. 

654 

655 @return pipe_base Struct containing these fields: 

656 - exposure calibrate science exposure with refined WCS and PhotoCalib 

657 - background model of background subtracted from exposure (an 

658 lsst.afw.math.BackgroundList) 

659 - sourceCat catalog of measured sources 

660 - astromMatches list of source/refObj matches from the astrometry 

661 solver 

662 """ 

663 # detect, deblend and measure sources 

664 if exposureIdInfo is None: 

665 exposureIdInfo = ExposureIdInfo() 

666 

667 if background is None: 

668 background = BackgroundList() 

669 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, 

670 exposureIdInfo.unusedBits) 

671 table = SourceTable.make(self.schema, sourceIdFactory) 

672 table.setMetadata(self.algMetadata) 

673 

674 detRes = self.detection.run(table=table, exposure=exposure, 

675 doSmooth=True) 

676 sourceCat = detRes.sources 

677 if detRes.fpSets.background: 677 ↛ 680line 677 didn't jump to line 680, because the condition on line 677 was never false

678 for bg in detRes.fpSets.background: 

679 background.append(bg) 

680 if self.config.doSkySources: 

681 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId) 

682 if skySourceFootprints: 682 ↛ 687line 682 didn't jump to line 687, because the condition on line 682 was never false

683 for foot in skySourceFootprints: 

684 s = sourceCat.addNew() 

685 s.setFootprint(foot) 

686 s.set(self.skySourceKey, True) 

687 if self.config.doDeblend: 687 ↛ 689line 687 didn't jump to line 689, because the condition on line 687 was never false

688 self.deblend.run(exposure=exposure, sources=sourceCat) 

689 self.measurement.run( 

690 measCat=sourceCat, 

691 exposure=exposure, 

692 exposureId=exposureIdInfo.expId 

693 ) 

694 if self.config.doApCorr: 

695 self.applyApCorr.run( 

696 catalog=sourceCat, 

697 apCorrMap=exposure.getInfo().getApCorrMap() 

698 ) 

699 self.catalogCalculation.run(sourceCat) 

700 

701 if icSourceCat is not None and \ 

702 len(self.config.icSourceFieldsToCopy) > 0: 

703 self.copyIcSourceFields(icSourceCat=icSourceCat, 

704 sourceCat=sourceCat) 

705 

706 # TODO DM-11568: this contiguous check-and-copy could go away if we 

707 # reserve enough space during SourceDetection and/or SourceDeblend. 

708 # NOTE: sourceSelectors require contiguous catalogs, so ensure 

709 # contiguity now, so views are preserved from here on. 

710 if not sourceCat.isContiguous(): 

711 sourceCat = sourceCat.copy(deep=True) 

712 

713 # perform astrometry calibration: 

714 # fit an improved WCS and update the exposure's WCS in place 

715 astromMatches = None 

716 matchMeta = None 

717 if self.config.doAstrometry: 717 ↛ 718line 717 didn't jump to line 718, because the condition on line 717 was never true

718 try: 

719 astromRes = self.astrometry.run( 

720 exposure=exposure, 

721 sourceCat=sourceCat, 

722 ) 

723 astromMatches = astromRes.matches 

724 matchMeta = astromRes.matchMeta 

725 except Exception as e: 

726 if self.config.requireAstrometry: 

727 raise 

728 self.log.warn("Unable to perform astrometric calibration " 

729 "(%s): attempting to proceed" % e) 

730 

731 # compute photometric calibration 

732 if self.config.doPhotoCal: 732 ↛ 733line 732 didn't jump to line 733, because the condition on line 732 was never true

733 try: 

734 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId) 

735 exposure.setPhotoCalib(photoRes.photoCalib) 

736 # TODO: reword this to phrase it in terms of the calibration factor? 

737 self.log.info("Photometric zero-point: %f" % 

738 photoRes.photoCalib.instFluxToMagnitude(1.0)) 

739 self.setMetadata(exposure=exposure, photoRes=photoRes) 

740 except Exception as e: 

741 if self.config.requirePhotoCal: 

742 raise 

743 self.log.warn("Unable to perform photometric calibration " 

744 "(%s): attempting to proceed" % e) 

745 self.setMetadata(exposure=exposure, photoRes=None) 

746 

747 if self.config.doInsertFakes: 

748 self.insertFakes.run(exposure, background=background) 

749 

750 table = SourceTable.make(self.schema, sourceIdFactory) 

751 table.setMetadata(self.algMetadata) 

752 

753 detRes = self.detection.run(table=table, exposure=exposure, 

754 doSmooth=True) 

755 sourceCat = detRes.sources 

756 if detRes.fpSets.background: 756 ↛ 759line 756 didn't jump to line 759, because the condition on line 756 was never false

757 for bg in detRes.fpSets.background: 

758 background.append(bg) 

759 if self.config.doDeblend: 759 ↛ 761line 759 didn't jump to line 761, because the condition on line 759 was never false

760 self.deblend.run(exposure=exposure, sources=sourceCat) 

761 self.measurement.run( 

762 measCat=sourceCat, 

763 exposure=exposure, 

764 exposureId=exposureIdInfo.expId 

765 ) 

766 if self.config.doApCorr: 766 ↛ 771line 766 didn't jump to line 771, because the condition on line 766 was never false

767 self.applyApCorr.run( 

768 catalog=sourceCat, 

769 apCorrMap=exposure.getInfo().getApCorrMap() 

770 ) 

771 self.catalogCalculation.run(sourceCat) 

772 

773 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0: 773 ↛ 777line 773 didn't jump to line 777, because the condition on line 773 was never false

774 self.copyIcSourceFields(icSourceCat=icSourceCat, 

775 sourceCat=sourceCat) 

776 

777 frame = getDebugFrame(self._display, "calibrate") 

778 if frame: 778 ↛ 779line 778 didn't jump to line 779, because the condition on line 778 was never true

779 displayAstrometry( 

780 sourceCat=sourceCat, 

781 exposure=exposure, 

782 matches=astromMatches, 

783 frame=frame, 

784 pause=False, 

785 ) 

786 

787 return pipeBase.Struct( 

788 exposure=exposure, 

789 background=background, 

790 sourceCat=sourceCat, 

791 astromMatches=astromMatches, 

792 matchMeta=matchMeta, 

793 # These are duplicate entries with different names for use with 

794 # gen3 middleware 

795 outputExposure=exposure, 

796 outputCat=sourceCat, 

797 outputBackground=background, 

798 ) 

799 

800 def writeOutputs(self, dataRef, exposure, background, sourceCat, 

801 astromMatches, matchMeta): 

802 """Write output data to the output repository 

803 

804 @param[in] dataRef butler data reference corresponding to a science 

805 image 

806 @param[in] exposure exposure to write 

807 @param[in] background background model for exposure 

808 @param[in] sourceCat catalog of measured sources 

809 @param[in] astromMatches list of source/refObj matches from the 

810 astrometry solver 

811 """ 

812 dataRef.put(sourceCat, "src") 

813 if self.config.doWriteMatches and astromMatches is not None: 813 ↛ 814line 813 didn't jump to line 814, because the condition on line 813 was never true

814 normalizedMatches = afwTable.packMatches(astromMatches) 

815 normalizedMatches.table.setMetadata(matchMeta) 

816 dataRef.put(normalizedMatches, "srcMatch") 

817 if self.config.doWriteMatchesDenormalized: 

818 denormMatches = denormalizeMatches(astromMatches, matchMeta) 

819 dataRef.put(denormMatches, "srcMatchFull") 

820 if self.config.doWriteExposure: 820 ↛ exitline 820 didn't return from function 'writeOutputs', because the condition on line 820 was never false

821 dataRef.put(exposure, "calexp") 

822 dataRef.put(background, "calexpBackground") 

823 

824 def getSchemaCatalogs(self): 

825 """Return a dict of empty catalogs for each catalog dataset produced 

826 by this task. 

827 """ 

828 sourceCat = afwTable.SourceCatalog(self.schema) 

829 sourceCat.getTable().setMetadata(self.algMetadata) 

830 return {"src": sourceCat} 

831 

832 def setMetadata(self, exposure, photoRes=None): 

833 """!Set task and exposure metadata 

834 

835 Logs a warning and continues if needed data is missing. 

836 

837 @param[in,out] exposure exposure whose metadata is to be set 

838 @param[in] photoRes results of running photoCal; if None then it was 

839 not run 

840 """ 

841 if photoRes is None: 

842 return 

843 

844 metadata = exposure.getMetadata() 

845 

846 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata 

847 try: 

848 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime() 

849 magZero = photoRes.zp - 2.5*math.log10(exposureTime) 

850 except Exception: 

851 self.log.warn("Could not set normalized MAGZERO in header: no " 

852 "exposure time") 

853 magZero = math.nan 

854 

855 try: 

856 metadata.set('MAGZERO', magZero) 

857 metadata.set('MAGZERO_RMS', photoRes.sigma) 

858 metadata.set('MAGZERO_NOBJ', photoRes.ngood) 

859 metadata.set('COLORTERM1', 0.0) 

860 metadata.set('COLORTERM2', 0.0) 

861 metadata.set('COLORTERM3', 0.0) 

862 except Exception as e: 

863 self.log.warn("Could not set exposure metadata: %s" % (e,)) 

864 

865 def copyIcSourceFields(self, icSourceCat, sourceCat): 

866 """!Match sources in icSourceCat and sourceCat and copy the specified fields 

867 

868 @param[in] icSourceCat catalog from which to copy fields 

869 @param[in,out] sourceCat catalog to which to copy fields 

870 

871 The fields copied are those specified by `config.icSourceFieldsToCopy` 

872 that actually exist in the schema. This was set up by the constructor 

873 using self.schemaMapper. 

874 """ 

875 if self.schemaMapper is None: 875 ↛ 876line 875 didn't jump to line 876, because the condition on line 875 was never true

876 raise RuntimeError("To copy icSource fields you must specify " 

877 "icSourceSchema nd icSourceKeys when " 

878 "constructing this task") 

879 if icSourceCat is None or sourceCat is None: 879 ↛ 880line 879 didn't jump to line 880, because the condition on line 879 was never true

880 raise RuntimeError("icSourceCat and sourceCat must both be " 

881 "specified") 

882 if len(self.config.icSourceFieldsToCopy) == 0: 882 ↛ 883line 882 didn't jump to line 883, because the condition on line 882 was never true

883 self.log.warn("copyIcSourceFields doing nothing because " 

884 "icSourceFieldsToCopy is empty") 

885 return 

886 

887 mc = afwTable.MatchControl() 

888 mc.findOnlyClosest = False # return all matched objects 

889 matches = afwTable.matchXy(icSourceCat, sourceCat, 

890 self.config.matchRadiusPix, mc) 

891 if self.config.doDeblend: 891 ↛ 900line 891 didn't jump to line 900, because the condition on line 891 was never false

892 deblendKey = sourceCat.schema["deblend_nChild"].asKey() 

893 # if deblended, keep children 

894 matches = [m for m in matches if m[1].get(deblendKey) == 0] 

895 

896 # Because we had to allow multiple matches to handle parents, we now 

897 # need to prune to the best matches 

898 # closest matches as a dict of icSourceCat source ID: 

899 # (icSourceCat source, sourceCat source, distance in pixels) 

900 bestMatches = {} 

901 for m0, m1, d in matches: 

902 id0 = m0.getId() 

903 match = bestMatches.get(id0) 

904 if match is None or d <= match[2]: 904 ↛ 901line 904 didn't jump to line 901, because the condition on line 904 was never false

905 bestMatches[id0] = (m0, m1, d) 

906 matches = list(bestMatches.values()) 

907 

908 # Check that no sourceCat sources are listed twice (we already know 

909 # that each match has a unique icSourceCat source ID, due to using 

910 # that ID as the key in bestMatches) 

911 numMatches = len(matches) 

912 numUniqueSources = len(set(m[1].getId() for m in matches)) 

913 if numUniqueSources != numMatches: 913 ↛ 914line 913 didn't jump to line 914, because the condition on line 913 was never true

914 self.log.warn("{} icSourceCat sources matched only {} sourceCat " 

915 "sources".format(numMatches, numUniqueSources)) 

916 

917 self.log.info("Copying flags from icSourceCat to sourceCat for " 

918 "%s sources" % (numMatches,)) 

919 

920 # For each match: set the calibSourceKey flag and copy the desired 

921 # fields 

922 for icSrc, src, d in matches: 

923 src.setFlag(self.calibSourceKey, True) 

924 # src.assign copies the footprint from icSrc, which we don't want 

925 # (DM-407) 

926 # so set icSrc's footprint to src's footprint before src.assign, 

927 # then restore it 

928 icSrcFootprint = icSrc.getFootprint() 

929 try: 

930 icSrc.setFootprint(src.getFootprint()) 

931 src.assign(icSrc, self.schemaMapper) 

932 finally: 

933 icSrc.setFootprint(icSrcFootprint)