Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34import sys 

35import traceback 

36import copy 

37 

38import numpy as np 

39import healpy as hp 

40import esutil 

41from astropy import units 

42 

43import lsst.daf.base as dafBase 

44import lsst.pex.config as pexConfig 

45import lsst.pipe.base as pipeBase 

46from lsst.pipe.base import connectionTypes 

47from lsst.afw.image import TransmissionCurve 

48from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

49from lsst.meas.algorithms import ReferenceObjectLoader 

50from lsst.pipe.tasks.photoCal import PhotoCalTask 

51import lsst.geom 

52import lsst.afw.image as afwImage 

53import lsst.afw.math as afwMath 

54import lsst.afw.table as afwTable 

55from lsst.meas.algorithms import IndexerRegistry 

56from lsst.meas.algorithms import DatasetConfig 

57from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

58 

59from .utilities import computeApproxPixelAreaFields 

60from .utilities import lookupStaticCalibrations 

61 

62import fgcm 

63 

64__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner'] 

65 

66 

67class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections, 

68 dimensions=("instrument",), 

69 defaultTemplates={"cycleNumber": "0"}): 

70 camera = connectionTypes.PrerequisiteInput( 

71 doc="Camera instrument", 

72 name="camera", 

73 storageClass="Camera", 

74 dimensions=("instrument",), 

75 lookupFunction=lookupStaticCalibrations, 

76 isCalibration=True, 

77 ) 

78 

79 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

80 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

81 "chromatic corrections."), 

82 name="fgcmLookUpTable", 

83 storageClass="Catalog", 

84 dimensions=("instrument",), 

85 deferLoad=True, 

86 ) 

87 

88 fgcmVisitCatalog = connectionTypes.PrerequisiteInput( 

89 doc="Catalog of visit information for fgcm", 

90 name="fgcmVisitCatalog", 

91 storageClass="Catalog", 

92 dimensions=("instrument",), 

93 deferLoad=True, 

94 ) 

95 

96 fgcmStandardStars = connectionTypes.PrerequisiteInput( 

97 doc="Catalog of standard star data from fgcm fit", 

98 name="fgcmStandardStars{cycleNumber}", 

99 storageClass="SimpleCatalog", 

100 dimensions=("instrument",), 

101 deferLoad=True, 

102 ) 

103 

104 fgcmZeropoints = connectionTypes.PrerequisiteInput( 

105 doc="Catalog of zeropoints from fgcm fit", 

106 name="fgcmZeropoints{cycleNumber}", 

107 storageClass="Catalog", 

108 dimensions=("instrument",), 

109 deferLoad=True, 

110 ) 

111 

112 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput( 

113 doc="Catalog of atmosphere parameters from fgcm fit", 

114 name="fgcmAtmosphereParameters{cycleNumber}", 

115 storageClass="Catalog", 

116 dimensions=("instrument",), 

117 deferLoad=True, 

118 ) 

119 

120 refCat = connectionTypes.PrerequisiteInput( 

121 doc="Reference catalog to use for photometric calibration", 

122 name="cal_ref_cat", 

123 storageClass="SimpleCatalog", 

124 dimensions=("skypix",), 

125 deferLoad=True, 

126 multiple=True, 

127 ) 

128 

129 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput( 

130 doc="Config used to build FGCM input stars", 

131 name="fgcmBuildStarsTable_config", 

132 storageClass="Config", 

133 ) 

134 

135 fgcmPhotoCalib = connectionTypes.Output( 

136 doc=("Per-visit photometric calibrations derived from fgcm calibration. " 

137 "These catalogs use detector id for the id and are sorted for " 

138 "fast lookups of a detector."), 

139 name="fgcmPhotoCalibCatalog", 

140 storageClass="ExposureCatalog", 

141 dimensions=("instrument", "visit",), 

142 multiple=True, 

143 ) 

144 

145 fgcmTransmissionAtmosphere = connectionTypes.Output( 

146 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

147 name="transmission_atmosphere_fgcm", 

148 storageClass="TransmissionCurve", 

149 dimensions=("instrument", 

150 "visit",), 

151 multiple=True, 

152 ) 

153 

154 fgcmOffsets = connectionTypes.Output( 

155 doc="Per-band offsets computed from doReferenceCalibration", 

156 name="fgcmReferenceCalibrationOffsets", 

157 storageClass="Catalog", 

158 dimensions=("instrument",), 

159 multiple=False, 

160 ) 

161 

162 def __init__(self, *, config=None): 

163 super().__init__(config=config) 

164 

165 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

166 raise ValueError("cycleNumber must be of integer format") 

167 if config.connections.refCat != config.refObjLoader.ref_dataset_name: 

168 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name") 

169 

170 if config.doRefcatOutput: 

171 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput") 

172 

173 if not config.doReferenceCalibration: 

174 self.prerequisiteInputs.remove("refCat") 

175 if not config.doAtmosphereOutput: 

176 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

177 if not config.doZeropointOutput: 

178 self.prerequisiteInputs.remove("fgcmZeropoints") 

179 if not config.doReferenceCalibration: 

180 self.outputs.remove("fgcmOffsets") 

181 

182 

183class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig, 

184 pipelineConnections=FgcmOutputProductsConnections): 

185 """Config for FgcmOutputProductsTask""" 

186 

187 cycleNumber = pexConfig.Field( 

188 doc="Final fit cycle from FGCM fit", 

189 dtype=int, 

190 default=None, 

191 ) 

192 

193 # The following fields refer to calibrating from a reference 

194 # catalog, but in the future this might need to be expanded 

195 doReferenceCalibration = pexConfig.Field( 

196 doc=("Transfer 'absolute' calibration from reference catalog? " 

197 "This afterburner step is unnecessary if reference stars " 

198 "were used in the full fit in FgcmFitCycleTask."), 

199 dtype=bool, 

200 default=False, 

201 ) 

202 doRefcatOutput = pexConfig.Field( 

203 doc="Output standard stars in reference catalog format", 

204 dtype=bool, 

205 default=True, 

206 ) 

207 doAtmosphereOutput = pexConfig.Field( 

208 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

209 dtype=bool, 

210 default=True, 

211 ) 

212 doZeropointOutput = pexConfig.Field( 

213 doc="Output zeropoints in fgcm_photoCalib format", 

214 dtype=bool, 

215 default=True, 

216 ) 

217 doComposeWcsJacobian = pexConfig.Field( 

218 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

219 dtype=bool, 

220 default=True, 

221 ) 

222 doApplyMeanChromaticCorrection = pexConfig.Field( 

223 doc="Apply the mean chromatic correction to the zeropoints?", 

224 dtype=bool, 

225 default=True, 

226 ) 

227 refObjLoader = pexConfig.ConfigurableField( 

228 target=LoadIndexedReferenceObjectsTask, 

229 doc="reference object loader for 'absolute' photometric calibration", 

230 ) 

231 photoCal = pexConfig.ConfigurableField( 

232 target=PhotoCalTask, 

233 doc="task to perform 'absolute' calibration", 

234 ) 

235 referencePixelizationNside = pexConfig.Field( 

236 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

237 dtype=int, 

238 default=64, 

239 ) 

240 referencePixelizationMinStars = pexConfig.Field( 

241 doc=("Minimum number of stars per healpix pixel to select for comparison" 

242 "to the specified reference catalog"), 

243 dtype=int, 

244 default=200, 

245 ) 

246 referenceMinMatch = pexConfig.Field( 

247 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

248 dtype=int, 

249 default=50, 

250 ) 

251 referencePixelizationNPixels = pexConfig.Field( 

252 doc=("Number of healpix pixels to sample to do comparison. " 

253 "Doing too many will take a long time and not yield any more " 

254 "precise results because the final number is the median offset " 

255 "(per band) from the set of pixels."), 

256 dtype=int, 

257 default=100, 

258 ) 

259 datasetConfig = pexConfig.ConfigField( 

260 dtype=DatasetConfig, 

261 doc="Configuration for writing/reading ingested catalog", 

262 ) 

263 

264 def setDefaults(self): 

265 pexConfig.Config.setDefaults(self) 

266 

267 # In order to transfer the "absolute" calibration from a reference 

268 # catalog to the relatively calibrated FGCM standard stars (one number 

269 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

270 # pixels. These basic settings ensure that only well-measured, good stars 

271 # from the source and reference catalogs are used for the matching. 

272 

273 # applyColorTerms needs to be False if doReferenceCalibration is False, 

274 # as is the new default after DM-16702 

275 self.photoCal.applyColorTerms = False 

276 self.photoCal.fluxField = 'instFlux' 

277 self.photoCal.magErrFloor = 0.003 

278 self.photoCal.match.referenceSelection.doSignalToNoise = True 

279 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

280 self.photoCal.match.sourceSelection.doSignalToNoise = True 

281 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

282 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

283 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

284 self.photoCal.match.sourceSelection.doFlags = True 

285 self.photoCal.match.sourceSelection.flags.good = [] 

286 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

287 self.photoCal.match.sourceSelection.doUnresolved = False 

288 self.datasetConfig.ref_dataset_name = 'fgcm_stars' 

289 self.datasetConfig.format_version = 1 

290 

291 def validate(self): 

292 super().validate() 

293 

294 # Force the connections to conform with cycleNumber 

295 self.connections.cycleNumber = str(self.cycleNumber) 

296 

297 

298class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner): 

299 """Subclass of TaskRunner for fgcmOutputProductsTask 

300 

301 fgcmOutputProductsTask.run() takes one argument, the butler, and 

302 does not run on any data in the repository. 

303 This runner does not use any parallelization. 

304 """ 

305 

306 @staticmethod 

307 def getTargetList(parsedCmd): 

308 """ 

309 Return a list with one element, the butler. 

310 """ 

311 return [parsedCmd.butler] 

312 

313 def __call__(self, butler): 

314 """ 

315 Parameters 

316 ---------- 

317 butler: `lsst.daf.persistence.Butler` 

318 

319 Returns 

320 ------- 

321 exitStatus: `list` with `pipeBase.Struct` 

322 exitStatus (0: success; 1: failure) 

323 if self.doReturnResults also 

324 results (`np.array` with absolute zeropoint offsets) 

325 """ 

326 task = self.TaskClass(butler=butler, config=self.config, log=self.log) 

327 

328 exitStatus = 0 

329 if self.doRaise: 

330 results = task.runDataRef(butler) 

331 else: 

332 try: 

333 results = task.runDataRef(butler) 

334 except Exception as e: 

335 exitStatus = 1 

336 task.log.fatal("Failed: %s" % e) 

337 if not isinstance(e, pipeBase.TaskError): 

338 traceback.print_exc(file=sys.stderr) 

339 

340 task.writeMetadata(butler) 

341 

342 if self.doReturnResults: 

343 # The results here are the zeropoint offsets for each band 

344 return [pipeBase.Struct(exitStatus=exitStatus, 

345 results=results)] 

346 else: 

347 return [pipeBase.Struct(exitStatus=exitStatus)] 

348 

349 def run(self, parsedCmd): 

350 """ 

351 Run the task, with no multiprocessing 

352 

353 Parameters 

354 ---------- 

355 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

356 """ 

357 

358 resultList = [] 

359 

360 if self.precall(parsedCmd): 

361 targetList = self.getTargetList(parsedCmd) 

362 # make sure that we only get 1 

363 resultList = self(targetList[0]) 

364 

365 return resultList 

366 

367 

368class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

369 """ 

370 Output products from FGCM global calibration. 

371 """ 

372 

373 ConfigClass = FgcmOutputProductsConfig 

374 RunnerClass = FgcmOutputProductsRunner 

375 _DefaultName = "fgcmOutputProducts" 

376 

377 def __init__(self, butler=None, **kwargs): 

378 super().__init__(**kwargs) 

379 

380 # no saving of metadata for now 

381 def _getMetadataName(self): 

382 return None 

383 

384 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

385 dataRefDict = {} 

386 dataRefDict['camera'] = butlerQC.get(inputRefs.camera) 

387 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

388 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

389 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars) 

390 

391 if self.config.doZeropointOutput: 

392 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints) 

393 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

394 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

395 

396 if self.config.doAtmosphereOutput: 

397 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters) 

398 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

399 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

400 

401 if self.config.doReferenceCalibration: 

402 refConfig = self.config.refObjLoader 

403 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

404 for ref in inputRefs.refCat], 

405 refCats=butlerQC.get(inputRefs.refCat), 

406 config=refConfig, 

407 log=self.log) 

408 else: 

409 self.refObjLoader = None 

410 

411 dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig) 

412 

413 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig) 

414 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

415 

416 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

417 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

418 "in fgcmBuildStarsTask.") 

419 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

420 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

421 

422 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True) 

423 

424 # Output the photoCalib exposure catalogs 

425 if struct.photoCalibCatalogs is not None: 

426 self.log.info("Outputting photoCalib catalogs.") 

427 for visit, expCatalog in struct.photoCalibCatalogs: 

428 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

429 self.log.info("Done outputting photoCalib catalogs.") 

430 

431 # Output the atmospheres 

432 if struct.atmospheres is not None: 

433 self.log.info("Outputting atmosphere transmission files.") 

434 for visit, atm in struct.atmospheres: 

435 butlerQC.put(atm, atmRefDict[visit]) 

436 self.log.info("Done outputting atmosphere files.") 

437 

438 if self.config.doReferenceCalibration: 

439 # Turn offset into simple catalog for persistence if necessary 

440 schema = afwTable.Schema() 

441 schema.addField('offset', type=np.float64, 

442 doc="Post-process calibration offset (mag)") 

443 offsetCat = afwTable.BaseCatalog(schema) 

444 offsetCat.resize(len(struct.offsets)) 

445 offsetCat['offset'][:] = struct.offsets 

446 

447 butlerQC.put(offsetCat, outputRefs.fgcmOffsets) 

448 

449 return 

450 

451 @pipeBase.timeMethod 

452 def runDataRef(self, butler): 

453 """ 

454 Make FGCM output products for use in the stack 

455 

456 Parameters 

457 ---------- 

458 butler: `lsst.daf.persistence.Butler` 

459 cycleNumber: `int` 

460 Final fit cycle number, override config. 

461 

462 Returns 

463 ------- 

464 offsets: `lsst.pipe.base.Struct` 

465 A structure with array of zeropoint offsets 

466 

467 Raises 

468 ------ 

469 RuntimeError: 

470 Raised if any one of the following is true: 

471 

472 - butler cannot find "fgcmBuildStars_config" or 

473 "fgcmBuildStarsTable_config". 

474 - butler cannot find "fgcmFitCycle_config". 

475 - "fgcmFitCycle_config" does not refer to 

476 `self.config.cycleNumber`. 

477 - butler cannot find "fgcmAtmosphereParameters" and 

478 `self.config.doAtmosphereOutput` is `True`. 

479 - butler cannot find "fgcmStandardStars" and 

480 `self.config.doReferenceCalibration` is `True` or 

481 `self.config.doRefcatOutput` is `True`. 

482 - butler cannot find "fgcmZeropoints" and 

483 `self.config.doZeropointOutput` is `True`. 

484 """ 

485 if self.config.doReferenceCalibration: 

486 # We need the ref obj loader to get the flux field 

487 self.makeSubtask("refObjLoader", butler=butler) 

488 

489 # Check to make sure that the fgcmBuildStars config exists, to retrieve 

490 # the visit and ccd dataset tags 

491 if not butler.datasetExists('fgcmBuildStarsTable_config') and \ 

492 not butler.datasetExists('fgcmBuildStars_config'): 

493 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, " 

494 "which is prereq for fgcmOutputProducts") 

495 

496 if butler.datasetExists('fgcmBuildStarsTable_config'): 

497 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config') 

498 else: 

499 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config') 

500 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

501 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

502 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

503 

504 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

505 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

506 "in fgcmBuildStarsTask.") 

507 

508 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

509 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

510 

511 # And make sure that the atmosphere was output properly 

512 if (self.config.doAtmosphereOutput 

513 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)): 

514 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.") 

515 

516 if not butler.datasetExists('fgcmStandardStars', 

517 fgcmcycle=self.config.cycleNumber): 

518 raise RuntimeError("Standard stars are missing for cycle %d." % 

519 (self.config.cycleNumber)) 

520 

521 if (self.config.doZeropointOutput 

522 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))): 

523 raise RuntimeError("Zeropoints are missing for cycle %d." % 

524 (self.config.cycleNumber)) 

525 

526 dataRefDict = {} 

527 # This is the _actual_ camera 

528 dataRefDict['camera'] = butler.get('camera') 

529 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

530 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

531 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars', 

532 fgcmcycle=self.config.cycleNumber) 

533 

534 if self.config.doZeropointOutput: 

535 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints', 

536 fgcmcycle=self.config.cycleNumber) 

537 if self.config.doAtmosphereOutput: 

538 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters', 

539 fgcmcycle=self.config.cycleNumber) 

540 

541 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False) 

542 

543 if struct.photoCalibs is not None: 

544 self.log.info("Outputting photoCalib files.") 

545 

546 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs: 

547 butler.put(photoCalib, 'fgcm_photoCalib', 

548 dataId={visitDataRefName: visit, 

549 ccdDataRefName: detector, 

550 'filter': physicalFilter}) 

551 

552 self.log.info("Done outputting photoCalib files.") 

553 

554 if struct.atmospheres is not None: 

555 self.log.info("Outputting atmosphere transmission files.") 

556 for visit, atm in struct.atmospheres: 

557 butler.put(atm, "transmission_atmosphere_fgcm", 

558 dataId={visitDataRefName: visit}) 

559 self.log.info("Done outputting atmosphere transmissions.") 

560 

561 return pipeBase.Struct(offsets=struct.offsets) 

562 

563 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None): 

564 """Run the output products task. 

565 

566 Parameters 

567 ---------- 

568 dataRefDict : `dict` 

569 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

570 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

571 dataRef dictionary with keys: 

572 

573 ``"camera"`` 

574 Camera object (`lsst.afw.cameraGeom.Camera`) 

575 ``"fgcmLookUpTable"`` 

576 dataRef for the FGCM look-up table. 

577 ``"fgcmVisitCatalog"`` 

578 dataRef for visit summary catalog. 

579 ``"fgcmStandardStars"`` 

580 dataRef for the output standard star catalog. 

581 ``"fgcmZeropoints"`` 

582 dataRef for the zeropoint data catalog. 

583 ``"fgcmAtmosphereParameters"`` 

584 dataRef for the atmosphere parameter catalog. 

585 ``"fgcmBuildStarsTableConfig"`` 

586 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`. 

587 physicalFilterMap : `dict` 

588 Dictionary of mappings from physical filter to FGCM band. 

589 returnCatalogs : `bool`, optional 

590 Return photoCalibs as per-visit exposure catalogs. 

591 butler : `lsst.daf.persistence.Butler`, optional 

592 Gen2 butler used for reference star outputs 

593 

594 Returns 

595 ------- 

596 retStruct : `lsst.pipe.base.Struct` 

597 Output structure with keys: 

598 

599 offsets : `np.ndarray` 

600 Final reference offsets, per band. 

601 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

602 Generator that returns (visit, transmissionCurve) tuples. 

603 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

604 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

605 (returned if returnCatalogs is False). 

606 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

607 Generator that returns (visit, exposureCatalog) tuples. 

608 (returned if returnCatalogs is True). 

609 """ 

610 stdCat = dataRefDict['fgcmStandardStars'].get() 

611 md = stdCat.getMetadata() 

612 bands = md.getArray('BANDS') 

613 

614 if self.config.doReferenceCalibration: 

615 lutCat = dataRefDict['fgcmLookUpTable'].get() 

616 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands) 

617 else: 

618 offsets = np.zeros(len(bands)) 

619 

620 # This is Gen2 only, and requires the butler. 

621 if self.config.doRefcatOutput and butler is not None: 

622 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig) 

623 

624 del stdCat 

625 

626 if self.config.doZeropointOutput: 

627 zptCat = dataRefDict['fgcmZeropoints'].get() 

628 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

629 

630 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands, 

631 physicalFilterMap, returnCatalogs=returnCatalogs) 

632 else: 

633 pcgen = None 

634 

635 if self.config.doAtmosphereOutput: 

636 atmCat = dataRefDict['fgcmAtmosphereParameters'].get() 

637 atmgen = self._outputAtmospheres(dataRefDict, atmCat) 

638 else: 

639 atmgen = None 

640 

641 retStruct = pipeBase.Struct(offsets=offsets, 

642 atmospheres=atmgen) 

643 if returnCatalogs: 

644 retStruct.photoCalibCatalogs = pcgen 

645 else: 

646 retStruct.photoCalibs = pcgen 

647 

648 return retStruct 

649 

650 def generateTractOutputProducts(self, dataRefDict, tract, 

651 visitCat, zptCat, atmCat, stdCat, 

652 fgcmBuildStarsConfig, 

653 returnCatalogs=True, 

654 butler=None): 

655 """ 

656 Generate the output products for a given tract, as specified in the config. 

657 

658 This method is here to have an alternate entry-point for 

659 FgcmCalibrateTract. 

660 

661 Parameters 

662 ---------- 

663 dataRefDict : `dict` 

664 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

665 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

666 dataRef dictionary with keys: 

667 

668 ``"camera"`` 

669 Camera object (`lsst.afw.cameraGeom.Camera`) 

670 ``"fgcmLookUpTable"`` 

671 dataRef for the FGCM look-up table. 

672 tract : `int` 

673 Tract number 

674 visitCat : `lsst.afw.table.BaseCatalog` 

675 FGCM visitCat from `FgcmBuildStarsTask` 

676 zptCat : `lsst.afw.table.BaseCatalog` 

677 FGCM zeropoint catalog from `FgcmFitCycleTask` 

678 atmCat : `lsst.afw.table.BaseCatalog` 

679 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

680 stdCat : `lsst.afw.table.SimpleCatalog` 

681 FGCM standard star catalog from `FgcmFitCycleTask` 

682 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig` 

683 Configuration object from `FgcmBuildStarsTask` 

684 returnCatalogs : `bool`, optional 

685 Return photoCalibs as per-visit exposure catalogs. 

686 butler: `lsst.daf.persistence.Butler`, optional 

687 Gen2 butler used for reference star outputs 

688 

689 Returns 

690 ------- 

691 retStruct : `lsst.pipe.base.Struct` 

692 Output structure with keys: 

693 

694 offsets : `np.ndarray` 

695 Final reference offsets, per band. 

696 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

697 Generator that returns (visit, transmissionCurve) tuples. 

698 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

699 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

700 (returned if returnCatalogs is False). 

701 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

702 Generator that returns (visit, exposureCatalog) tuples. 

703 (returned if returnCatalogs is True). 

704 """ 

705 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

706 

707 md = stdCat.getMetadata() 

708 bands = md.getArray('BANDS') 

709 

710 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

711 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

712 "in fgcmBuildStarsTask.") 

713 

714 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

715 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

716 

717 if self.config.doReferenceCalibration: 

718 lutCat = dataRefDict['fgcmLookUpTable'].get() 

719 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap) 

720 else: 

721 offsets = np.zeros(len(bands)) 

722 

723 if self.config.doRefcatOutput and butler is not None: 

724 # Create a special config that has the tract number in it 

725 datasetConfig = copy.copy(self.config.datasetConfig) 

726 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name, 

727 tract) 

728 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig) 

729 

730 if self.config.doZeropointOutput: 

731 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands, 

732 physicalFilterMap, returnCatalogs=returnCatalogs) 

733 else: 

734 pcgen = None 

735 

736 if self.config.doAtmosphereOutput: 

737 atmgen = self._outputAtmospheres(dataRefDict, atmCat) 

738 else: 

739 atmgen = None 

740 

741 retStruct = pipeBase.Struct(offsets=offsets, 

742 atmospheres=atmgen) 

743 if returnCatalogs: 

744 retStruct.photoCalibCatalogs = pcgen 

745 else: 

746 retStruct.photoCalibs = pcgen 

747 

748 return retStruct 

749 

750 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands): 

751 """ 

752 Compute offsets relative to a reference catalog. 

753 

754 This method splits the star catalog into healpix pixels 

755 and computes the calibration transfer for a sample of 

756 these pixels to approximate the 'absolute' calibration 

757 values (on for each band) to apply to transfer the 

758 absolute scale. 

759 

760 Parameters 

761 ---------- 

762 stdCat : `lsst.afw.table.SimpleCatalog` 

763 FGCM standard stars 

764 lutCat : `lsst.afw.table.SimpleCatalog` 

765 FGCM Look-up table 

766 physicalFilterMap : `dict` 

767 Dictionary of mappings from physical filter to FGCM band. 

768 bands : `list` [`str`] 

769 List of band names from FGCM output 

770 Returns 

771 ------- 

772 offsets : `numpy.array` of floats 

773 Per band zeropoint offsets 

774 """ 

775 

776 # Only use stars that are observed in all the bands that were actually used 

777 # This will ensure that we use the same healpix pixels for the absolute 

778 # calibration of each band. 

779 minObs = stdCat['ngood'].min(axis=1) 

780 

781 goodStars = (minObs >= 1) 

782 stdCat = stdCat[goodStars] 

783 

784 self.log.info("Found %d stars with at least 1 good observation in each band" % 

785 (len(stdCat))) 

786 

787 # Associate each band with the appropriate physicalFilter and make 

788 # filterLabels 

789 filterLabels = [] 

790 

791 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',') 

792 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',') 

793 physicalFilterMapBands = list(physicalFilterMap.values()) 

794 physicalFilterMapFilters = list(physicalFilterMap.keys()) 

795 for band in bands: 

796 # Find a physical filter associated from the band by doing 

797 # a reverse lookup on the physicalFilterMap dict 

798 physicalFilterMapIndex = physicalFilterMapBands.index(band) 

799 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex] 

800 # Find the appropriate fgcm standard physicalFilter 

801 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter) 

802 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex] 

803 filterLabels.append(afwImage.FilterLabel(band=band, 

804 physical=stdPhysicalFilter)) 

805 

806 # We have to make a table for each pixel with flux/fluxErr 

807 # This is a temporary table generated for input to the photoCal task. 

808 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

809 # have had chromatic corrections applied to get to the standard system 

810 # specified by the atmosphere/instrumental parameters), nor are they 

811 # in Jansky (since they don't have a proper absolute calibration: the overall 

812 # zeropoint is estimated from the telescope size, etc.) 

813 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

814 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

815 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

816 doc="instrumental flux (counts)") 

817 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

818 doc="instrumental flux error (counts)") 

819 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

820 type='Flag', 

821 doc="bad flag") 

822 

823 # Split up the stars 

824 # Note that there is an assumption here that the ra/dec coords stored 

825 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

826 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

827 theta = np.pi/2. - stdCat['coord_dec'] 

828 phi = stdCat['coord_ra'] 

829 

830 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) 

831 h, rev = esutil.stat.histogram(ipring, rev=True) 

832 

833 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

834 

835 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

836 (gdpix.size, 

837 self.config.referencePixelizationNside, 

838 self.config.referencePixelizationMinStars)) 

839 

840 if gdpix.size < self.config.referencePixelizationNPixels: 

841 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

842 (gdpix.size, self.config.referencePixelizationNPixels)) 

843 else: 

844 # Sample out the pixels we want to use 

845 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

846 

847 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

848 ('nstar', 'i4', len(bands)), 

849 ('nmatch', 'i4', len(bands)), 

850 ('zp', 'f4', len(bands)), 

851 ('zpErr', 'f4', len(bands))]) 

852 results['hpix'] = ipring[rev[rev[gdpix]]] 

853 

854 # We need a boolean index to deal with catalogs... 

855 selected = np.zeros(len(stdCat), dtype=bool) 

856 

857 refFluxFields = [None]*len(bands) 

858 

859 for p_index, pix in enumerate(gdpix): 

860 i1a = rev[rev[pix]: rev[pix + 1]] 

861 

862 # the stdCat afwTable can only be indexed with boolean arrays, 

863 # and not numpy index arrays (see DM-16497). This little trick 

864 # converts the index array into a boolean array 

865 selected[:] = False 

866 selected[i1a] = True 

867 

868 for b_index, filterLabel in enumerate(filterLabels): 

869 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index, 

870 filterLabel, stdCat, 

871 selected, refFluxFields) 

872 results['nstar'][p_index, b_index] = len(i1a) 

873 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag) 

874 results['zp'][p_index, b_index] = struct.zp 

875 results['zpErr'][p_index, b_index] = struct.sigma 

876 

877 # And compute the summary statistics 

878 offsets = np.zeros(len(bands)) 

879 

880 for b_index, band in enumerate(bands): 

881 # make configurable 

882 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch) 

883 offsets[b_index] = np.median(results['zp'][ok, b_index]) 

884 # use median absolute deviation to estimate Normal sigma 

885 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

886 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index])) 

887 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f", 

888 band, offsets[b_index], madSigma) 

889 

890 return offsets 

891 

892 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

893 b_index, filterLabel, stdCat, selected, refFluxFields): 

894 """ 

895 Compute the zeropoint offset between the fgcm stdCat and the reference 

896 stars for one pixel in one band 

897 

898 Parameters 

899 ---------- 

900 sourceMapper : `lsst.afw.table.SchemaMapper` 

901 Mapper to go from stdCat to calibratable catalog 

902 badStarKey : `lsst.afw.table.Key` 

903 Key for the field with bad stars 

904 b_index : `int` 

905 Index of the band in the star catalog 

906 filterLabel : `lsst.afw.image.FilterLabel` 

907 filterLabel with band and physical filter 

908 stdCat : `lsst.afw.table.SimpleCatalog` 

909 FGCM standard stars 

910 selected : `numpy.array(dtype=bool)` 

911 Boolean array of which stars are in the pixel 

912 refFluxFields : `list` 

913 List of names of flux fields for reference catalog 

914 """ 

915 

916 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

917 sourceCat.reserve(selected.sum()) 

918 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

919 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5)) 

920 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index] 

921 * sourceCat['instFlux']) 

922 # Make sure we only use stars that have valid measurements 

923 # (This is perhaps redundant with requirements above that the 

924 # stars be observed in all bands, but it can't hurt) 

925 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0) 

926 for rec in sourceCat[badStar]: 

927 rec.set(badStarKey, True) 

928 

929 exposure = afwImage.ExposureF() 

930 exposure.setFilterLabel(filterLabel) 

931 

932 if refFluxFields[b_index] is None: 

933 # Need to find the flux field in the reference catalog 

934 # to work around limitations of DirectMatch in PhotoCal 

935 ctr = stdCat[0].getCoord() 

936 rad = 0.05*lsst.geom.degrees 

937 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel) 

938 refFluxFields[b_index] = refDataTest.fluxField 

939 

940 # Make a copy of the config so that we can modify it 

941 calConfig = copy.copy(self.config.photoCal.value) 

942 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index] 

943 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err' 

944 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

945 config=calConfig, 

946 schema=sourceCat.getSchema()) 

947 

948 struct = calTask.run(exposure, sourceCat) 

949 

950 return struct 

951 

952 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig): 

953 """ 

954 Output standard stars in indexed reference catalog format. 

955 This is not currently supported in Gen3. 

956 

957 Parameters 

958 ---------- 

959 butler : `lsst.daf.persistence.Butler` 

960 stdCat : `lsst.afw.table.SimpleCatalog` 

961 FGCM standard star catalog from fgcmFitCycleTask 

962 offsets : `numpy.array` of floats 

963 Per band zeropoint offsets 

964 bands : `list` [`str`] 

965 List of band names from FGCM output 

966 datasetConfig : `lsst.meas.algorithms.DatasetConfig` 

967 Config for reference dataset 

968 """ 

969 

970 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name)) 

971 

972 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name]( 

973 self.config.datasetConfig.indexer.active) 

974 

975 # We determine the conversion from the native units (typically radians) to 

976 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

977 # numpy arrays rather than Angles, which would we approximately 600x slower. 

978 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords 

979 # (as Angles) for input 

980 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra']) 

981 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv, 

982 stdCat['coord_dec']*conv)) 

983 

984 formattedCat = self._formatCatalog(stdCat, offsets, bands) 

985 

986 # Write the master schema 

987 dataId = indexer.makeDataId('master_schema', 

988 datasetConfig.ref_dataset_name) 

989 masterCat = afwTable.SimpleCatalog(formattedCat.schema) 

990 addRefCatMetadata(masterCat) 

991 butler.put(masterCat, 'ref_cat', dataId=dataId) 

992 

993 # Break up the pixels using a histogram 

994 h, rev = esutil.stat.histogram(indices, rev=True) 

995 gd, = np.where(h > 0) 

996 selected = np.zeros(len(formattedCat), dtype=bool) 

997 for i in gd: 

998 i1a = rev[rev[i]: rev[i + 1]] 

999 

1000 # the formattedCat afwTable can only be indexed with boolean arrays, 

1001 # and not numpy index arrays (see DM-16497). This little trick 

1002 # converts the index array into a boolean array 

1003 selected[:] = False 

1004 selected[i1a] = True 

1005 

1006 # Write the individual pixel 

1007 dataId = indexer.makeDataId(indices[i1a[0]], 

1008 datasetConfig.ref_dataset_name) 

1009 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId) 

1010 

1011 # And save the dataset configuration 

1012 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name) 

1013 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId) 

1014 

1015 self.log.info("Done outputting standard stars.") 

1016 

1017 def _formatCatalog(self, fgcmStarCat, offsets, bands): 

1018 """ 

1019 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

1020 

1021 Parameters 

1022 ---------- 

1023 fgcmStarCat : `lsst.afw.Table.SimpleCatalog` 

1024 SimpleCatalog as output by fgcmcal 

1025 offsets : `list` with len(self.bands) entries 

1026 Zeropoint offsets to apply 

1027 bands : `list` [`str`] 

1028 List of band names from FGCM output 

1029 

1030 Returns 

1031 ------- 

1032 formattedCat: `lsst.afw.table.SimpleCatalog` 

1033 SimpleCatalog suitable for using as a reference catalog 

1034 """ 

1035 

1036 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

1037 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands, 

1038 addCentroid=False, 

1039 addIsResolved=True, 

1040 coordErrDim=0) 

1041 sourceMapper.addMinimalSchema(minSchema) 

1042 for band in bands: 

1043 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

1044 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

1045 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

1046 

1047 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

1048 formattedCat.reserve(len(fgcmStarCat)) 

1049 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

1050 

1051 # Note that we don't have to set `resolved` because the default is False 

1052 

1053 for b, band in enumerate(bands): 

1054 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

1055 # We want fluxes in nJy from calibrated AB magnitudes 

1056 # (after applying offset). Updated after RFC-549 and RFC-575. 

1057 flux = (mag*units.ABmag).to_value(units.nJy) 

1058 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

1059 

1060 formattedCat['%s_flux' % (band)][:] = flux 

1061 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

1062 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

1063 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

1064 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

1065 

1066 addRefCatMetadata(formattedCat) 

1067 

1068 return formattedCat 

1069 

1070 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands, 

1071 physicalFilterMap, returnCatalogs=True, 

1072 tract=None): 

1073 """Output the zeropoints in fgcm_photoCalib format. 

1074 

1075 Parameters 

1076 ---------- 

1077 camera : `lsst.afw.cameraGeom.Camera` 

1078 Camera from the butler. 

1079 zptCat : `lsst.afw.table.BaseCatalog` 

1080 FGCM zeropoint catalog from `FgcmFitCycleTask`. 

1081 visitCat : `lsst.afw.table.BaseCatalog` 

1082 FGCM visitCat from `FgcmBuildStarsTask`. 

1083 offsets : `numpy.array` 

1084 Float array of absolute calibration offsets, one for each filter. 

1085 bands : `list` [`str`] 

1086 List of band names from FGCM output. 

1087 physicalFilterMap : `dict` 

1088 Dictionary of mappings from physical filter to FGCM band. 

1089 returnCatalogs : `bool`, optional 

1090 Return photoCalibs as per-visit exposure catalogs. 

1091 tract: `int`, optional 

1092 Tract number to output. Default is None (global calibration) 

1093 

1094 Returns 

1095 ------- 

1096 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

1097 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

1098 (returned if returnCatalogs is False). 

1099 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

1100 Generator that returns (visit, exposureCatalog) tuples. 

1101 (returned if returnCatalogs is True). 

1102 """ 

1103 # Select visit/ccds where we have a calibration 

1104 # This includes ccds where we were able to interpolate from neighboring 

1105 # ccds. 

1106 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

1107 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) 

1108 & (zptCat['fgcmZptVar'] > 0.0)) 

1109 

1110 # Log warnings for any visit which has no valid zeropoints 

1111 badVisits = np.unique(zptCat['visit'][~selected]) 

1112 goodVisits = np.unique(zptCat['visit'][selected]) 

1113 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

1114 for allBadVisit in allBadVisits: 

1115 self.log.warn(f'No suitable photoCalib for visit {allBadVisit}') 

1116 

1117 # Get a mapping from filtername to the offsets 

1118 offsetMapping = {} 

1119 for f in physicalFilterMap: 

1120 # Not every filter in the map will necesarily have a band. 

1121 if physicalFilterMap[f] in bands: 

1122 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])] 

1123 

1124 # Get a mapping from "ccd" to the ccd index used for the scaling 

1125 ccdMapping = {} 

1126 for ccdIndex, detector in enumerate(camera): 

1127 ccdMapping[detector.getId()] = ccdIndex 

1128 

1129 # And a mapping to get the flat-field scaling values 

1130 scalingMapping = {} 

1131 for rec in visitCat: 

1132 scalingMapping[rec['visit']] = rec['scaling'] 

1133 

1134 if self.config.doComposeWcsJacobian: 

1135 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

1136 

1137 # The zptCat is sorted by visit, which is useful 

1138 lastVisit = -1 

1139 zptVisitCatalog = None 

1140 

1141 metadata = dafBase.PropertyList() 

1142 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

1143 metadata.add("COMMENT", "Only detectors with data have entries.") 

1144 

1145 for rec in zptCat[selected]: 

1146 # Retrieve overall scaling 

1147 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]] 

1148 

1149 # The postCalibrationOffset describe any zeropoint offsets 

1150 # to apply after the fgcm calibration. The first part comes 

1151 # from the reference catalog match (used in testing). The 

1152 # second part comes from the mean chromatic correction 

1153 # (if configured). 

1154 postCalibrationOffset = offsetMapping[rec['filtername']] 

1155 if self.config.doApplyMeanChromaticCorrection: 

1156 postCalibrationOffset += rec['fgcmDeltaChrom'] 

1157 

1158 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

1159 rec['fgcmfZptChebXyMax']) 

1160 # Convert from FGCM AB to nJy 

1161 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

1162 rec['fgcmfZptChebXyMax'], 

1163 offset=postCalibrationOffset, 

1164 scaling=scaling) 

1165 

1166 if self.config.doComposeWcsJacobian: 

1167 

1168 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']], 

1169 fgcmSuperStarField, 

1170 fgcmZptField]) 

1171 else: 

1172 # The photoCalib is just the product of the fgcmSuperStarField and the 

1173 # fgcmZptField 

1174 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

1175 

1176 # The "mean" calibration will be set to the center of the ccd for reference 

1177 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

1178 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

1179 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

1180 calibrationErr=calibErr, 

1181 calibration=fgcmField, 

1182 isConstant=False) 

1183 

1184 if not returnCatalogs: 

1185 # Return individual photoCalibs 

1186 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib) 

1187 else: 

1188 # Return full per-visit exposure catalogs 

1189 if rec['visit'] != lastVisit: 

1190 # This is a new visit. If the last visit was not -1, yield 

1191 # the ExposureCatalog 

1192 if lastVisit > -1: 

1193 # ensure that the detectors are in sorted order, for fast lookups 

1194 zptVisitCatalog.sort() 

1195 yield (int(lastVisit), zptVisitCatalog) 

1196 else: 

1197 # We need to create a new schema 

1198 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema() 

1199 zptExpCatSchema.addField('visit', type='I', doc='Visit number') 

1200 

1201 # And start a new one 

1202 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema) 

1203 zptVisitCatalog.setMetadata(metadata) 

1204 

1205 lastVisit = int(rec['visit']) 

1206 

1207 catRecord = zptVisitCatalog.addNew() 

1208 catRecord['id'] = int(rec['detector']) 

1209 catRecord['visit'] = rec['visit'] 

1210 catRecord.setPhotoCalib(photoCalib) 

1211 

1212 # Final output of last exposure catalog 

1213 if returnCatalogs: 

1214 # ensure that the detectors are in sorted order, for fast lookups 

1215 zptVisitCatalog.sort() 

1216 yield (int(lastVisit), zptVisitCatalog) 

1217 

1218 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

1219 """ 

1220 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

1221 and scaling. 

1222 

1223 Parameters 

1224 ---------- 

1225 coefficients: `numpy.array` 

1226 Flattened array of chebyshev coefficients 

1227 xyMax: `list` of length 2 

1228 Maximum x and y of the chebyshev bounding box 

1229 offset: `float`, optional 

1230 Absolute calibration offset. Default is 0.0 

1231 scaling: `float`, optional 

1232 Flat scaling value from fgcmBuildStars. Default is 1.0 

1233 

1234 Returns 

1235 ------- 

1236 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

1237 """ 

1238 

1239 orderPlus1 = int(np.sqrt(coefficients.size)) 

1240 pars = np.zeros((orderPlus1, orderPlus1)) 

1241 

1242 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

1243 lsst.geom.Point2I(*xyMax)) 

1244 

1245 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) 

1246 * (10.**(offset/-2.5))*scaling) 

1247 

1248 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

1249 

1250 return boundedField 

1251 

1252 def _outputAtmospheres(self, dataRefDict, atmCat): 

1253 """ 

1254 Output the atmospheres. 

1255 

1256 Parameters 

1257 ---------- 

1258 dataRefDict : `dict` 

1259 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1260 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1261 dataRef dictionary with keys: 

1262 

1263 ``"fgcmLookUpTable"`` 

1264 dataRef for the FGCM look-up table. 

1265 atmCat : `lsst.afw.table.BaseCatalog` 

1266 FGCM atmosphere parameter catalog from fgcmFitCycleTask. 

1267 

1268 Returns 

1269 ------- 

1270 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

1271 Generator that returns (visit, transmissionCurve) tuples. 

1272 """ 

1273 # First, we need to grab the look-up table and key info 

1274 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1275 

1276 atmosphereTableName = lutCat[0]['tablename'] 

1277 elevation = lutCat[0]['elevation'] 

1278 atmLambda = lutCat[0]['atmLambda'] 

1279 lutCat = None 

1280 

1281 # Make the atmosphere table if possible 

1282 try: 

1283 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

1284 atmTable.loadTable() 

1285 except IOError: 

1286 atmTable = None 

1287 

1288 if atmTable is None: 

1289 # Try to use MODTRAN instead 

1290 try: 

1291 modGen = fgcm.ModtranGenerator(elevation) 

1292 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

1293 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

1294 except (ValueError, IOError) as e: 

1295 raise RuntimeError("FGCM look-up-table generated with modtran, " 

1296 "but modtran not configured to run.") from e 

1297 

1298 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

1299 

1300 for i, visit in enumerate(atmCat['visit']): 

1301 if atmTable is not None: 

1302 # Interpolate the atmosphere table 

1303 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

1304 pwv=atmCat[i]['pwv'], 

1305 o3=atmCat[i]['o3'], 

1306 tau=atmCat[i]['tau'], 

1307 alpha=atmCat[i]['alpha'], 

1308 zenith=zenith[i], 

1309 ctranslamstd=[atmCat[i]['cTrans'], 

1310 atmCat[i]['lamStd']]) 

1311 else: 

1312 # Run modtran 

1313 modAtm = modGen(pmb=atmCat[i]['pmb'], 

1314 pwv=atmCat[i]['pwv'], 

1315 o3=atmCat[i]['o3'], 

1316 tau=atmCat[i]['tau'], 

1317 alpha=atmCat[i]['alpha'], 

1318 zenith=zenith[i], 

1319 lambdaRange=lambdaRange, 

1320 lambdaStep=lambdaStep, 

1321 ctranslamstd=[atmCat[i]['cTrans'], 

1322 atmCat[i]['lamStd']]) 

1323 atmVals = modAtm['COMBINED'] 

1324 

1325 # Now need to create something to persist... 

1326 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

1327 wavelengths=atmLambda, 

1328 throughputAtMin=atmVals[0], 

1329 throughputAtMax=atmVals[-1]) 

1330 

1331 yield (int(visit), curve)