Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34import sys 

35import traceback 

36import copy 

37 

38import numpy as np 

39import healpy as hp 

40import esutil 

41from astropy import units 

42 

43import lsst.daf.base as dafBase 

44import lsst.pex.config as pexConfig 

45import lsst.pipe.base as pipeBase 

46from lsst.pipe.base import connectionTypes 

47from lsst.afw.image import TransmissionCurve 

48from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

49from lsst.meas.algorithms import ReferenceObjectLoader 

50from lsst.pipe.tasks.photoCal import PhotoCalTask 

51import lsst.geom 

52import lsst.afw.image as afwImage 

53import lsst.afw.math as afwMath 

54import lsst.afw.table as afwTable 

55from lsst.meas.algorithms import IndexerRegistry 

56from lsst.meas.algorithms import DatasetConfig 

57from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

58 

59from .utilities import computeApproxPixelAreaFields 

60from .utilities import lookupStaticCalibrations 

61 

62import fgcm 

63 

64__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner'] 

65 

66 

67class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections, 

68 dimensions=("instrument",), 

69 defaultTemplates={"cycleNumber": "0"}): 

70 camera = connectionTypes.PrerequisiteInput( 

71 doc="Camera instrument", 

72 name="camera", 

73 storageClass="Camera", 

74 dimensions=("instrument",), 

75 lookupFunction=lookupStaticCalibrations, 

76 isCalibration=True, 

77 ) 

78 

79 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

80 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

81 "chromatic corrections."), 

82 name="fgcmLookUpTable", 

83 storageClass="Catalog", 

84 dimensions=("instrument",), 

85 deferLoad=True, 

86 ) 

87 

88 fgcmVisitCatalog = connectionTypes.Input( 

89 doc="Catalog of visit information for fgcm", 

90 name="fgcmVisitCatalog", 

91 storageClass="Catalog", 

92 dimensions=("instrument",), 

93 deferLoad=True, 

94 ) 

95 

96 fgcmStandardStars = connectionTypes.Input( 

97 doc="Catalog of standard star data from fgcm fit", 

98 name="fgcmStandardStars{cycleNumber}", 

99 storageClass="SimpleCatalog", 

100 dimensions=("instrument",), 

101 deferLoad=True, 

102 ) 

103 

104 fgcmZeropoints = connectionTypes.Input( 

105 doc="Catalog of zeropoints from fgcm fit", 

106 name="fgcmZeropoints{cycleNumber}", 

107 storageClass="Catalog", 

108 dimensions=("instrument",), 

109 deferLoad=True, 

110 ) 

111 

112 fgcmAtmosphereParameters = connectionTypes.Input( 

113 doc="Catalog of atmosphere parameters from fgcm fit", 

114 name="fgcmAtmosphereParameters{cycleNumber}", 

115 storageClass="Catalog", 

116 dimensions=("instrument",), 

117 deferLoad=True, 

118 ) 

119 

120 refCat = connectionTypes.PrerequisiteInput( 

121 doc="Reference catalog to use for photometric calibration", 

122 name="cal_ref_cat", 

123 storageClass="SimpleCatalog", 

124 dimensions=("skypix",), 

125 deferLoad=True, 

126 multiple=True, 

127 ) 

128 

129 fgcmPhotoCalib = connectionTypes.Output( 

130 doc=("Per-visit photometric calibrations derived from fgcm calibration. " 

131 "These catalogs use detector id for the id and are sorted for " 

132 "fast lookups of a detector."), 

133 name="fgcmPhotoCalibCatalog", 

134 storageClass="ExposureCatalog", 

135 dimensions=("instrument", "visit",), 

136 multiple=True, 

137 ) 

138 

139 fgcmTransmissionAtmosphere = connectionTypes.Output( 

140 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

141 name="transmission_atmosphere_fgcm", 

142 storageClass="TransmissionCurve", 

143 dimensions=("instrument", 

144 "visit",), 

145 multiple=True, 

146 ) 

147 

148 fgcmOffsets = connectionTypes.Output( 

149 doc="Per-band offsets computed from doReferenceCalibration", 

150 name="fgcmReferenceCalibrationOffsets", 

151 storageClass="Catalog", 

152 dimensions=("instrument",), 

153 multiple=False, 

154 ) 

155 

156 def __init__(self, *, config=None): 

157 super().__init__(config=config) 

158 

159 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

160 raise ValueError("cycleNumber must be of integer format") 

161 if config.connections.refCat != config.refObjLoader.ref_dataset_name: 

162 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name") 

163 

164 if config.doRefcatOutput: 

165 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput") 

166 

167 if not config.doReferenceCalibration: 

168 self.prerequisiteInputs.remove("refCat") 

169 if not config.doAtmosphereOutput: 

170 self.inputs.remove("fgcmAtmosphereParameters") 

171 if not config.doZeropointOutput: 

172 self.inputs.remove("fgcmZeropoints") 

173 if not config.doReferenceCalibration: 

174 self.outputs.remove("fgcmOffsets") 

175 

176 

177class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig, 

178 pipelineConnections=FgcmOutputProductsConnections): 

179 """Config for FgcmOutputProductsTask""" 

180 

181 cycleNumber = pexConfig.Field( 

182 doc="Final fit cycle from FGCM fit", 

183 dtype=int, 

184 default=None, 

185 ) 

186 physicalFilterMap = pexConfig.DictField( 

187 doc="Mapping from 'physicalFilter' to band.", 

188 keytype=str, 

189 itemtype=str, 

190 default={}, 

191 ) 

192 # The following fields refer to calibrating from a reference 

193 # catalog, but in the future this might need to be expanded 

194 doReferenceCalibration = pexConfig.Field( 

195 doc=("Transfer 'absolute' calibration from reference catalog? " 

196 "This afterburner step is unnecessary if reference stars " 

197 "were used in the full fit in FgcmFitCycleTask."), 

198 dtype=bool, 

199 default=False, 

200 ) 

201 doRefcatOutput = pexConfig.Field( 

202 doc="Output standard stars in reference catalog format", 

203 dtype=bool, 

204 default=True, 

205 ) 

206 doAtmosphereOutput = pexConfig.Field( 

207 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

208 dtype=bool, 

209 default=True, 

210 ) 

211 doZeropointOutput = pexConfig.Field( 

212 doc="Output zeropoints in fgcm_photoCalib format", 

213 dtype=bool, 

214 default=True, 

215 ) 

216 doComposeWcsJacobian = pexConfig.Field( 

217 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

218 dtype=bool, 

219 default=True, 

220 ) 

221 doApplyMeanChromaticCorrection = pexConfig.Field( 

222 doc="Apply the mean chromatic correction to the zeropoints?", 

223 dtype=bool, 

224 default=True, 

225 ) 

226 refObjLoader = pexConfig.ConfigurableField( 

227 target=LoadIndexedReferenceObjectsTask, 

228 doc="reference object loader for 'absolute' photometric calibration", 

229 ) 

230 photoCal = pexConfig.ConfigurableField( 

231 target=PhotoCalTask, 

232 doc="task to perform 'absolute' calibration", 

233 ) 

234 referencePixelizationNside = pexConfig.Field( 

235 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

236 dtype=int, 

237 default=64, 

238 ) 

239 referencePixelizationMinStars = pexConfig.Field( 

240 doc=("Minimum number of stars per healpix pixel to select for comparison" 

241 "to the specified reference catalog"), 

242 dtype=int, 

243 default=200, 

244 ) 

245 referenceMinMatch = pexConfig.Field( 

246 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

247 dtype=int, 

248 default=50, 

249 ) 

250 referencePixelizationNPixels = pexConfig.Field( 

251 doc=("Number of healpix pixels to sample to do comparison. " 

252 "Doing too many will take a long time and not yield any more " 

253 "precise results because the final number is the median offset " 

254 "(per band) from the set of pixels."), 

255 dtype=int, 

256 default=100, 

257 ) 

258 datasetConfig = pexConfig.ConfigField( 

259 dtype=DatasetConfig, 

260 doc="Configuration for writing/reading ingested catalog", 

261 ) 

262 

263 def setDefaults(self): 

264 pexConfig.Config.setDefaults(self) 

265 

266 # In order to transfer the "absolute" calibration from a reference 

267 # catalog to the relatively calibrated FGCM standard stars (one number 

268 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

269 # pixels. These basic settings ensure that only well-measured, good stars 

270 # from the source and reference catalogs are used for the matching. 

271 

272 # applyColorTerms needs to be False if doReferenceCalibration is False, 

273 # as is the new default after DM-16702 

274 self.photoCal.applyColorTerms = False 

275 self.photoCal.fluxField = 'instFlux' 

276 self.photoCal.magErrFloor = 0.003 

277 self.photoCal.match.referenceSelection.doSignalToNoise = True 

278 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

279 self.photoCal.match.sourceSelection.doSignalToNoise = True 

280 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

281 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

282 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

283 self.photoCal.match.sourceSelection.doFlags = True 

284 self.photoCal.match.sourceSelection.flags.good = [] 

285 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

286 self.photoCal.match.sourceSelection.doUnresolved = False 

287 self.datasetConfig.ref_dataset_name = 'fgcm_stars' 

288 self.datasetConfig.format_version = 1 

289 

290 def validate(self): 

291 super().validate() 

292 

293 # Force the connections to conform with cycleNumber 

294 self.connections.cycleNumber = str(self.cycleNumber) 

295 

296 

297class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner): 

298 """Subclass of TaskRunner for fgcmOutputProductsTask 

299 

300 fgcmOutputProductsTask.run() takes one argument, the butler, and 

301 does not run on any data in the repository. 

302 This runner does not use any parallelization. 

303 """ 

304 

305 @staticmethod 

306 def getTargetList(parsedCmd): 

307 """ 

308 Return a list with one element, the butler. 

309 """ 

310 return [parsedCmd.butler] 

311 

312 def __call__(self, butler): 

313 """ 

314 Parameters 

315 ---------- 

316 butler: `lsst.daf.persistence.Butler` 

317 

318 Returns 

319 ------- 

320 exitStatus: `list` with `pipeBase.Struct` 

321 exitStatus (0: success; 1: failure) 

322 if self.doReturnResults also 

323 results (`np.array` with absolute zeropoint offsets) 

324 """ 

325 task = self.TaskClass(butler=butler, config=self.config, log=self.log) 

326 

327 exitStatus = 0 

328 if self.doRaise: 

329 results = task.runDataRef(butler) 

330 else: 

331 try: 

332 results = task.runDataRef(butler) 

333 except Exception as e: 

334 exitStatus = 1 

335 task.log.fatal("Failed: %s" % e) 

336 if not isinstance(e, pipeBase.TaskError): 

337 traceback.print_exc(file=sys.stderr) 

338 

339 task.writeMetadata(butler) 

340 

341 if self.doReturnResults: 

342 # The results here are the zeropoint offsets for each band 

343 return [pipeBase.Struct(exitStatus=exitStatus, 

344 results=results)] 

345 else: 

346 return [pipeBase.Struct(exitStatus=exitStatus)] 

347 

348 def run(self, parsedCmd): 

349 """ 

350 Run the task, with no multiprocessing 

351 

352 Parameters 

353 ---------- 

354 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

355 """ 

356 

357 resultList = [] 

358 

359 if self.precall(parsedCmd): 

360 targetList = self.getTargetList(parsedCmd) 

361 # make sure that we only get 1 

362 resultList = self(targetList[0]) 

363 

364 return resultList 

365 

366 

367class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

368 """ 

369 Output products from FGCM global calibration. 

370 """ 

371 

372 ConfigClass = FgcmOutputProductsConfig 

373 RunnerClass = FgcmOutputProductsRunner 

374 _DefaultName = "fgcmOutputProducts" 

375 

376 def __init__(self, butler=None, **kwargs): 

377 super().__init__(**kwargs) 

378 

379 # no saving of metadata for now 

380 def _getMetadataName(self): 

381 return None 

382 

383 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

384 dataRefDict = {} 

385 dataRefDict['camera'] = butlerQC.get(inputRefs.camera) 

386 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

387 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

388 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars) 

389 

390 if self.config.doZeropointOutput: 

391 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints) 

392 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

393 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

394 

395 if self.config.doAtmosphereOutput: 

396 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters) 

397 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

398 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

399 

400 if self.config.doReferenceCalibration: 

401 refConfig = self.config.refObjLoader 

402 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

403 for ref in inputRefs.refCat], 

404 refCats=butlerQC.get(inputRefs.refCat), 

405 config=refConfig, 

406 log=self.log) 

407 else: 

408 self.refObjLoader = None 

409 

410 struct = self.run(dataRefDict, self.config.physicalFilterMap, returnCatalogs=True) 

411 

412 # Output the photoCalib exposure catalogs 

413 if struct.photoCalibCatalogs is not None: 

414 self.log.info("Outputting photoCalib catalogs.") 

415 for visit, expCatalog in struct.photoCalibCatalogs: 

416 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

417 self.log.info("Done outputting photoCalib catalogs.") 

418 

419 # Output the atmospheres 

420 if struct.atmospheres is not None: 

421 self.log.info("Outputting atmosphere transmission files.") 

422 for visit, atm in struct.atmospheres: 

423 butlerQC.put(atm, atmRefDict[visit]) 

424 self.log.info("Done outputting atmosphere files.") 

425 

426 if self.config.doReferenceCalibration: 

427 # Turn offset into simple catalog for persistence if necessary 

428 schema = afwTable.Schema() 

429 schema.addField('offset', type=np.float64, 

430 doc="Post-process calibration offset (mag)") 

431 offsetCat = afwTable.BaseCatalog(schema) 

432 offsetCat.resize(len(struct.offsets)) 

433 offsetCat['offset'][:] = struct.offsets 

434 

435 butlerQC.put(offsetCat, outputRefs.fgcmOffsets) 

436 

437 return 

438 

439 @pipeBase.timeMethod 

440 def runDataRef(self, butler): 

441 """ 

442 Make FGCM output products for use in the stack 

443 

444 Parameters 

445 ---------- 

446 butler: `lsst.daf.persistence.Butler` 

447 cycleNumber: `int` 

448 Final fit cycle number, override config. 

449 

450 Returns 

451 ------- 

452 offsets: `lsst.pipe.base.Struct` 

453 A structure with array of zeropoint offsets 

454 

455 Raises 

456 ------ 

457 RuntimeError: 

458 Raised if any one of the following is true: 

459 

460 - butler cannot find "fgcmBuildStars_config" or 

461 "fgcmBuildStarsTable_config". 

462 - butler cannot find "fgcmFitCycle_config". 

463 - "fgcmFitCycle_config" does not refer to 

464 `self.config.cycleNumber`. 

465 - butler cannot find "fgcmAtmosphereParameters" and 

466 `self.config.doAtmosphereOutput` is `True`. 

467 - butler cannot find "fgcmStandardStars" and 

468 `self.config.doReferenceCalibration` is `True` or 

469 `self.config.doRefcatOutput` is `True`. 

470 - butler cannot find "fgcmZeropoints" and 

471 `self.config.doZeropointOutput` is `True`. 

472 """ 

473 if self.config.doReferenceCalibration: 

474 # We need the ref obj loader to get the flux field 

475 self.makeSubtask("refObjLoader", butler=butler) 

476 

477 # Check to make sure that the fgcmBuildStars config exists, to retrieve 

478 # the visit and ccd dataset tags 

479 if not butler.datasetExists('fgcmBuildStarsTable_config') and \ 

480 not butler.datasetExists('fgcmBuildStars_config'): 

481 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, " 

482 "which is prereq for fgcmOutputProducts") 

483 

484 if butler.datasetExists('fgcmBuildStarsTable_config'): 

485 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config') 

486 else: 

487 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config') 

488 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

489 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

490 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

491 

492 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

493 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

494 "in fgcmBuildStarsTask.") 

495 

496 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

497 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

498 

499 # And make sure that the atmosphere was output properly 

500 if (self.config.doAtmosphereOutput 

501 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)): 

502 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.") 

503 

504 if not butler.datasetExists('fgcmStandardStars', 

505 fgcmcycle=self.config.cycleNumber): 

506 raise RuntimeError("Standard stars are missing for cycle %d." % 

507 (self.config.cycleNumber)) 

508 

509 if (self.config.doZeropointOutput 

510 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))): 

511 raise RuntimeError("Zeropoints are missing for cycle %d." % 

512 (self.config.cycleNumber)) 

513 

514 dataRefDict = {} 

515 # This is the _actual_ camera 

516 dataRefDict['camera'] = butler.get('camera') 

517 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

518 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

519 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars', 

520 fgcmcycle=self.config.cycleNumber) 

521 

522 if self.config.doZeropointOutput: 

523 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints', 

524 fgcmcycle=self.config.cycleNumber) 

525 if self.config.doAtmosphereOutput: 

526 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters', 

527 fgcmcycle=self.config.cycleNumber) 

528 

529 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False) 

530 

531 if struct.photoCalibs is not None: 

532 self.log.info("Outputting photoCalib files.") 

533 

534 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs: 

535 butler.put(photoCalib, 'fgcm_photoCalib', 

536 dataId={visitDataRefName: visit, 

537 ccdDataRefName: detector, 

538 'filter': physicalFilter}) 

539 

540 self.log.info("Done outputting photoCalib files.") 

541 

542 if struct.atmospheres is not None: 

543 self.log.info("Outputting atmosphere transmission files.") 

544 for visit, atm in struct.atmospheres: 

545 butler.put(atm, "transmission_atmosphere_fgcm", 

546 dataId={visitDataRefName: visit}) 

547 self.log.info("Done outputting atmosphere transmissions.") 

548 

549 return pipeBase.Struct(offsets=struct.offsets) 

550 

551 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None): 

552 """Run the output products task. 

553 

554 Parameters 

555 ---------- 

556 dataRefDict : `dict` 

557 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

558 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

559 dataRef dictionary with keys: 

560 

561 ``"camera"`` 

562 Camera object (`lsst.afw.cameraGeom.Camera`) 

563 ``"fgcmLookUpTable"`` 

564 dataRef for the FGCM look-up table. 

565 ``"fgcmVisitCatalog"`` 

566 dataRef for visit summary catalog. 

567 ``"fgcmStandardStars"`` 

568 dataRef for the output standard star catalog. 

569 ``"fgcmZeropoints"`` 

570 dataRef for the zeropoint data catalog. 

571 ``"fgcmAtmosphereParameters"`` 

572 dataRef for the atmosphere parameter catalog. 

573 ``"fgcmBuildStarsTableConfig"`` 

574 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`. 

575 physicalFilterMap : `dict` 

576 Dictionary of mappings from physical filter to FGCM band. 

577 returnCatalogs : `bool`, optional 

578 Return photoCalibs as per-visit exposure catalogs. 

579 butler : `lsst.daf.persistence.Butler`, optional 

580 Gen2 butler used for reference star outputs 

581 

582 Returns 

583 ------- 

584 retStruct : `lsst.pipe.base.Struct` 

585 Output structure with keys: 

586 

587 offsets : `np.ndarray` 

588 Final reference offsets, per band. 

589 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

590 Generator that returns (visit, transmissionCurve) tuples. 

591 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

592 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

593 (returned if returnCatalogs is False). 

594 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

595 Generator that returns (visit, exposureCatalog) tuples. 

596 (returned if returnCatalogs is True). 

597 """ 

598 stdCat = dataRefDict['fgcmStandardStars'].get() 

599 md = stdCat.getMetadata() 

600 bands = md.getArray('BANDS') 

601 

602 if self.config.doReferenceCalibration: 

603 lutCat = dataRefDict['fgcmLookUpTable'].get() 

604 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands) 

605 else: 

606 offsets = np.zeros(len(bands)) 

607 

608 # This is Gen2 only, and requires the butler. 

609 if self.config.doRefcatOutput and butler is not None: 

610 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig) 

611 

612 del stdCat 

613 

614 if self.config.doZeropointOutput: 

615 zptCat = dataRefDict['fgcmZeropoints'].get() 

616 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

617 

618 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands, 

619 physicalFilterMap, returnCatalogs=returnCatalogs) 

620 else: 

621 pcgen = None 

622 

623 if self.config.doAtmosphereOutput: 

624 atmCat = dataRefDict['fgcmAtmosphereParameters'].get() 

625 atmgen = self._outputAtmospheres(dataRefDict, atmCat) 

626 else: 

627 atmgen = None 

628 

629 retStruct = pipeBase.Struct(offsets=offsets, 

630 atmospheres=atmgen) 

631 if returnCatalogs: 

632 retStruct.photoCalibCatalogs = pcgen 

633 else: 

634 retStruct.photoCalibs = pcgen 

635 

636 return retStruct 

637 

638 def generateTractOutputProducts(self, dataRefDict, tract, 

639 visitCat, zptCat, atmCat, stdCat, 

640 fgcmBuildStarsConfig, 

641 returnCatalogs=True, 

642 butler=None): 

643 """ 

644 Generate the output products for a given tract, as specified in the config. 

645 

646 This method is here to have an alternate entry-point for 

647 FgcmCalibrateTract. 

648 

649 Parameters 

650 ---------- 

651 dataRefDict : `dict` 

652 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

653 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

654 dataRef dictionary with keys: 

655 

656 ``"camera"`` 

657 Camera object (`lsst.afw.cameraGeom.Camera`) 

658 ``"fgcmLookUpTable"`` 

659 dataRef for the FGCM look-up table. 

660 tract : `int` 

661 Tract number 

662 visitCat : `lsst.afw.table.BaseCatalog` 

663 FGCM visitCat from `FgcmBuildStarsTask` 

664 zptCat : `lsst.afw.table.BaseCatalog` 

665 FGCM zeropoint catalog from `FgcmFitCycleTask` 

666 atmCat : `lsst.afw.table.BaseCatalog` 

667 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

668 stdCat : `lsst.afw.table.SimpleCatalog` 

669 FGCM standard star catalog from `FgcmFitCycleTask` 

670 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig` 

671 Configuration object from `FgcmBuildStarsTask` 

672 returnCatalogs : `bool`, optional 

673 Return photoCalibs as per-visit exposure catalogs. 

674 butler: `lsst.daf.persistence.Butler`, optional 

675 Gen2 butler used for reference star outputs 

676 

677 Returns 

678 ------- 

679 retStruct : `lsst.pipe.base.Struct` 

680 Output structure with keys: 

681 

682 offsets : `np.ndarray` 

683 Final reference offsets, per band. 

684 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

685 Generator that returns (visit, transmissionCurve) tuples. 

686 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

687 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

688 (returned if returnCatalogs is False). 

689 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

690 Generator that returns (visit, exposureCatalog) tuples. 

691 (returned if returnCatalogs is True). 

692 """ 

693 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

694 

695 md = stdCat.getMetadata() 

696 bands = md.getArray('BANDS') 

697 

698 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

699 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

700 "in fgcmBuildStarsTask.") 

701 

702 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

703 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

704 

705 if self.config.doReferenceCalibration: 

706 lutCat = dataRefDict['fgcmLookUpTable'].get() 

707 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap) 

708 else: 

709 offsets = np.zeros(len(bands)) 

710 

711 if self.config.doRefcatOutput and butler is not None: 

712 # Create a special config that has the tract number in it 

713 datasetConfig = copy.copy(self.config.datasetConfig) 

714 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name, 

715 tract) 

716 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig) 

717 

718 if self.config.doZeropointOutput: 

719 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands, 

720 physicalFilterMap, returnCatalogs=returnCatalogs) 

721 else: 

722 pcgen = None 

723 

724 if self.config.doAtmosphereOutput: 

725 atmgen = self._outputAtmospheres(dataRefDict, atmCat) 

726 else: 

727 atmgen = None 

728 

729 retStruct = pipeBase.Struct(offsets=offsets, 

730 atmospheres=atmgen) 

731 if returnCatalogs: 

732 retStruct.photoCalibCatalogs = pcgen 

733 else: 

734 retStruct.photoCalibs = pcgen 

735 

736 return retStruct 

737 

738 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands): 

739 """ 

740 Compute offsets relative to a reference catalog. 

741 

742 This method splits the star catalog into healpix pixels 

743 and computes the calibration transfer for a sample of 

744 these pixels to approximate the 'absolute' calibration 

745 values (on for each band) to apply to transfer the 

746 absolute scale. 

747 

748 Parameters 

749 ---------- 

750 stdCat : `lsst.afw.table.SimpleCatalog` 

751 FGCM standard stars 

752 lutCat : `lsst.afw.table.SimpleCatalog` 

753 FGCM Look-up table 

754 physicalFilterMap : `dict` 

755 Dictionary of mappings from physical filter to FGCM band. 

756 bands : `list` [`str`] 

757 List of band names from FGCM output 

758 Returns 

759 ------- 

760 offsets : `numpy.array` of floats 

761 Per band zeropoint offsets 

762 """ 

763 

764 # Only use stars that are observed in all the bands that were actually used 

765 # This will ensure that we use the same healpix pixels for the absolute 

766 # calibration of each band. 

767 minObs = stdCat['ngood'].min(axis=1) 

768 

769 goodStars = (minObs >= 1) 

770 stdCat = stdCat[goodStars] 

771 

772 self.log.info("Found %d stars with at least 1 good observation in each band" % 

773 (len(stdCat))) 

774 

775 # Associate each band with the appropriate physicalFilter and make 

776 # filterLabels 

777 filterLabels = [] 

778 

779 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',') 

780 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',') 

781 physicalFilterMapBands = list(physicalFilterMap.values()) 

782 physicalFilterMapFilters = list(physicalFilterMap.keys()) 

783 for band in bands: 

784 # Find a physical filter associated from the band by doing 

785 # a reverse lookup on the physicalFilterMap dict 

786 physicalFilterMapIndex = physicalFilterMapBands.index(band) 

787 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex] 

788 # Find the appropriate fgcm standard physicalFilter 

789 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter) 

790 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex] 

791 filterLabels.append(afwImage.FilterLabel(band=band, 

792 physical=stdPhysicalFilter)) 

793 

794 # We have to make a table for each pixel with flux/fluxErr 

795 # This is a temporary table generated for input to the photoCal task. 

796 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

797 # have had chromatic corrections applied to get to the standard system 

798 # specified by the atmosphere/instrumental parameters), nor are they 

799 # in Jansky (since they don't have a proper absolute calibration: the overall 

800 # zeropoint is estimated from the telescope size, etc.) 

801 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

802 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

803 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

804 doc="instrumental flux (counts)") 

805 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

806 doc="instrumental flux error (counts)") 

807 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

808 type='Flag', 

809 doc="bad flag") 

810 

811 # Split up the stars 

812 # Note that there is an assumption here that the ra/dec coords stored 

813 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

814 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

815 theta = np.pi/2. - stdCat['coord_dec'] 

816 phi = stdCat['coord_ra'] 

817 

818 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) 

819 h, rev = esutil.stat.histogram(ipring, rev=True) 

820 

821 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

822 

823 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

824 (gdpix.size, 

825 self.config.referencePixelizationNside, 

826 self.config.referencePixelizationMinStars)) 

827 

828 if gdpix.size < self.config.referencePixelizationNPixels: 

829 self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

830 (gdpix.size, self.config.referencePixelizationNPixels)) 

831 else: 

832 # Sample out the pixels we want to use 

833 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

834 

835 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

836 ('nstar', 'i4', len(bands)), 

837 ('nmatch', 'i4', len(bands)), 

838 ('zp', 'f4', len(bands)), 

839 ('zpErr', 'f4', len(bands))]) 

840 results['hpix'] = ipring[rev[rev[gdpix]]] 

841 

842 # We need a boolean index to deal with catalogs... 

843 selected = np.zeros(len(stdCat), dtype=bool) 

844 

845 refFluxFields = [None]*len(bands) 

846 

847 for p_index, pix in enumerate(gdpix): 

848 i1a = rev[rev[pix]: rev[pix + 1]] 

849 

850 # the stdCat afwTable can only be indexed with boolean arrays, 

851 # and not numpy index arrays (see DM-16497). This little trick 

852 # converts the index array into a boolean array 

853 selected[:] = False 

854 selected[i1a] = True 

855 

856 for b_index, filterLabel in enumerate(filterLabels): 

857 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index, 

858 filterLabel, stdCat, 

859 selected, refFluxFields) 

860 results['nstar'][p_index, b_index] = len(i1a) 

861 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag) 

862 results['zp'][p_index, b_index] = struct.zp 

863 results['zpErr'][p_index, b_index] = struct.sigma 

864 

865 # And compute the summary statistics 

866 offsets = np.zeros(len(bands)) 

867 

868 for b_index, band in enumerate(bands): 

869 # make configurable 

870 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch) 

871 offsets[b_index] = np.median(results['zp'][ok, b_index]) 

872 # use median absolute deviation to estimate Normal sigma 

873 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

874 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index])) 

875 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f", 

876 band, offsets[b_index], madSigma) 

877 

878 return offsets 

879 

880 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

881 b_index, filterLabel, stdCat, selected, refFluxFields): 

882 """ 

883 Compute the zeropoint offset between the fgcm stdCat and the reference 

884 stars for one pixel in one band 

885 

886 Parameters 

887 ---------- 

888 sourceMapper : `lsst.afw.table.SchemaMapper` 

889 Mapper to go from stdCat to calibratable catalog 

890 badStarKey : `lsst.afw.table.Key` 

891 Key for the field with bad stars 

892 b_index : `int` 

893 Index of the band in the star catalog 

894 filterLabel : `lsst.afw.image.FilterLabel` 

895 filterLabel with band and physical filter 

896 stdCat : `lsst.afw.table.SimpleCatalog` 

897 FGCM standard stars 

898 selected : `numpy.array(dtype=bool)` 

899 Boolean array of which stars are in the pixel 

900 refFluxFields : `list` 

901 List of names of flux fields for reference catalog 

902 """ 

903 

904 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

905 sourceCat.reserve(selected.sum()) 

906 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

907 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5)) 

908 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index] 

909 * sourceCat['instFlux']) 

910 # Make sure we only use stars that have valid measurements 

911 # (This is perhaps redundant with requirements above that the 

912 # stars be observed in all bands, but it can't hurt) 

913 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0) 

914 for rec in sourceCat[badStar]: 

915 rec.set(badStarKey, True) 

916 

917 exposure = afwImage.ExposureF() 

918 exposure.setFilterLabel(filterLabel) 

919 

920 if refFluxFields[b_index] is None: 

921 # Need to find the flux field in the reference catalog 

922 # to work around limitations of DirectMatch in PhotoCal 

923 ctr = stdCat[0].getCoord() 

924 rad = 0.05*lsst.geom.degrees 

925 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel) 

926 refFluxFields[b_index] = refDataTest.fluxField 

927 

928 # Make a copy of the config so that we can modify it 

929 calConfig = copy.copy(self.config.photoCal.value) 

930 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index] 

931 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err' 

932 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

933 config=calConfig, 

934 schema=sourceCat.getSchema()) 

935 

936 struct = calTask.run(exposure, sourceCat) 

937 

938 return struct 

939 

940 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig): 

941 """ 

942 Output standard stars in indexed reference catalog format. 

943 This is not currently supported in Gen3. 

944 

945 Parameters 

946 ---------- 

947 butler : `lsst.daf.persistence.Butler` 

948 stdCat : `lsst.afw.table.SimpleCatalog` 

949 FGCM standard star catalog from fgcmFitCycleTask 

950 offsets : `numpy.array` of floats 

951 Per band zeropoint offsets 

952 bands : `list` [`str`] 

953 List of band names from FGCM output 

954 datasetConfig : `lsst.meas.algorithms.DatasetConfig` 

955 Config for reference dataset 

956 """ 

957 

958 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name)) 

959 

960 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name]( 

961 self.config.datasetConfig.indexer.active) 

962 

963 # We determine the conversion from the native units (typically radians) to 

964 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

965 # numpy arrays rather than Angles, which would we approximately 600x slower. 

966 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords 

967 # (as Angles) for input 

968 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra']) 

969 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv, 

970 stdCat['coord_dec']*conv)) 

971 

972 formattedCat = self._formatCatalog(stdCat, offsets, bands) 

973 

974 # Write the master schema 

975 dataId = indexer.makeDataId('master_schema', 

976 datasetConfig.ref_dataset_name) 

977 masterCat = afwTable.SimpleCatalog(formattedCat.schema) 

978 addRefCatMetadata(masterCat) 

979 butler.put(masterCat, 'ref_cat', dataId=dataId) 

980 

981 # Break up the pixels using a histogram 

982 h, rev = esutil.stat.histogram(indices, rev=True) 

983 gd, = np.where(h > 0) 

984 selected = np.zeros(len(formattedCat), dtype=bool) 

985 for i in gd: 

986 i1a = rev[rev[i]: rev[i + 1]] 

987 

988 # the formattedCat afwTable can only be indexed with boolean arrays, 

989 # and not numpy index arrays (see DM-16497). This little trick 

990 # converts the index array into a boolean array 

991 selected[:] = False 

992 selected[i1a] = True 

993 

994 # Write the individual pixel 

995 dataId = indexer.makeDataId(indices[i1a[0]], 

996 datasetConfig.ref_dataset_name) 

997 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId) 

998 

999 # And save the dataset configuration 

1000 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name) 

1001 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId) 

1002 

1003 self.log.info("Done outputting standard stars.") 

1004 

1005 def _formatCatalog(self, fgcmStarCat, offsets, bands): 

1006 """ 

1007 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

1008 

1009 Parameters 

1010 ---------- 

1011 fgcmStarCat : `lsst.afw.Table.SimpleCatalog` 

1012 SimpleCatalog as output by fgcmcal 

1013 offsets : `list` with len(self.bands) entries 

1014 Zeropoint offsets to apply 

1015 bands : `list` [`str`] 

1016 List of band names from FGCM output 

1017 

1018 Returns 

1019 ------- 

1020 formattedCat: `lsst.afw.table.SimpleCatalog` 

1021 SimpleCatalog suitable for using as a reference catalog 

1022 """ 

1023 

1024 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

1025 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands, 

1026 addCentroid=False, 

1027 addIsResolved=True, 

1028 coordErrDim=0) 

1029 sourceMapper.addMinimalSchema(minSchema) 

1030 for band in bands: 

1031 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

1032 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

1033 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

1034 

1035 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

1036 formattedCat.reserve(len(fgcmStarCat)) 

1037 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

1038 

1039 # Note that we don't have to set `resolved` because the default is False 

1040 

1041 for b, band in enumerate(bands): 

1042 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

1043 # We want fluxes in nJy from calibrated AB magnitudes 

1044 # (after applying offset). Updated after RFC-549 and RFC-575. 

1045 flux = (mag*units.ABmag).to_value(units.nJy) 

1046 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

1047 

1048 formattedCat['%s_flux' % (band)][:] = flux 

1049 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

1050 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

1051 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

1052 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

1053 

1054 addRefCatMetadata(formattedCat) 

1055 

1056 return formattedCat 

1057 

1058 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands, 

1059 physicalFilterMap, returnCatalogs=True, 

1060 tract=None): 

1061 """Output the zeropoints in fgcm_photoCalib format. 

1062 

1063 Parameters 

1064 ---------- 

1065 camera : `lsst.afw.cameraGeom.Camera` 

1066 Camera from the butler. 

1067 zptCat : `lsst.afw.table.BaseCatalog` 

1068 FGCM zeropoint catalog from `FgcmFitCycleTask`. 

1069 visitCat : `lsst.afw.table.BaseCatalog` 

1070 FGCM visitCat from `FgcmBuildStarsTask`. 

1071 offsets : `numpy.array` 

1072 Float array of absolute calibration offsets, one for each filter. 

1073 bands : `list` [`str`] 

1074 List of band names from FGCM output. 

1075 physicalFilterMap : `dict` 

1076 Dictionary of mappings from physical filter to FGCM band. 

1077 returnCatalogs : `bool`, optional 

1078 Return photoCalibs as per-visit exposure catalogs. 

1079 tract: `int`, optional 

1080 Tract number to output. Default is None (global calibration) 

1081 

1082 Returns 

1083 ------- 

1084 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

1085 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

1086 (returned if returnCatalogs is False). 

1087 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

1088 Generator that returns (visit, exposureCatalog) tuples. 

1089 (returned if returnCatalogs is True). 

1090 """ 

1091 # Select visit/ccds where we have a calibration 

1092 # This includes ccds where we were able to interpolate from neighboring 

1093 # ccds. 

1094 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

1095 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) 

1096 & (zptCat['fgcmZptVar'] > 0.0)) 

1097 

1098 # Log warnings for any visit which has no valid zeropoints 

1099 badVisits = np.unique(zptCat['visit'][~selected]) 

1100 goodVisits = np.unique(zptCat['visit'][selected]) 

1101 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

1102 for allBadVisit in allBadVisits: 

1103 self.log.warning(f'No suitable photoCalib for visit {allBadVisit}') 

1104 

1105 # Get a mapping from filtername to the offsets 

1106 offsetMapping = {} 

1107 for f in physicalFilterMap: 

1108 # Not every filter in the map will necesarily have a band. 

1109 if physicalFilterMap[f] in bands: 

1110 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])] 

1111 

1112 # Get a mapping from "ccd" to the ccd index used for the scaling 

1113 ccdMapping = {} 

1114 for ccdIndex, detector in enumerate(camera): 

1115 ccdMapping[detector.getId()] = ccdIndex 

1116 

1117 # And a mapping to get the flat-field scaling values 

1118 scalingMapping = {} 

1119 for rec in visitCat: 

1120 scalingMapping[rec['visit']] = rec['scaling'] 

1121 

1122 if self.config.doComposeWcsJacobian: 

1123 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

1124 

1125 # The zptCat is sorted by visit, which is useful 

1126 lastVisit = -1 

1127 zptVisitCatalog = None 

1128 

1129 metadata = dafBase.PropertyList() 

1130 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

1131 metadata.add("COMMENT", "Only detectors with data have entries.") 

1132 

1133 for rec in zptCat[selected]: 

1134 # Retrieve overall scaling 

1135 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]] 

1136 

1137 # The postCalibrationOffset describe any zeropoint offsets 

1138 # to apply after the fgcm calibration. The first part comes 

1139 # from the reference catalog match (used in testing). The 

1140 # second part comes from the mean chromatic correction 

1141 # (if configured). 

1142 postCalibrationOffset = offsetMapping[rec['filtername']] 

1143 if self.config.doApplyMeanChromaticCorrection: 

1144 postCalibrationOffset += rec['fgcmDeltaChrom'] 

1145 

1146 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

1147 rec['fgcmfZptChebXyMax']) 

1148 # Convert from FGCM AB to nJy 

1149 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

1150 rec['fgcmfZptChebXyMax'], 

1151 offset=postCalibrationOffset, 

1152 scaling=scaling) 

1153 

1154 if self.config.doComposeWcsJacobian: 

1155 

1156 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']], 

1157 fgcmSuperStarField, 

1158 fgcmZptField]) 

1159 else: 

1160 # The photoCalib is just the product of the fgcmSuperStarField and the 

1161 # fgcmZptField 

1162 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

1163 

1164 # The "mean" calibration will be set to the center of the ccd for reference 

1165 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

1166 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

1167 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

1168 calibrationErr=calibErr, 

1169 calibration=fgcmField, 

1170 isConstant=False) 

1171 

1172 if not returnCatalogs: 

1173 # Return individual photoCalibs 

1174 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib) 

1175 else: 

1176 # Return full per-visit exposure catalogs 

1177 if rec['visit'] != lastVisit: 

1178 # This is a new visit. If the last visit was not -1, yield 

1179 # the ExposureCatalog 

1180 if lastVisit > -1: 

1181 # ensure that the detectors are in sorted order, for fast lookups 

1182 zptVisitCatalog.sort() 

1183 yield (int(lastVisit), zptVisitCatalog) 

1184 else: 

1185 # We need to create a new schema 

1186 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema() 

1187 zptExpCatSchema.addField('visit', type='I', doc='Visit number') 

1188 

1189 # And start a new one 

1190 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema) 

1191 zptVisitCatalog.setMetadata(metadata) 

1192 

1193 lastVisit = int(rec['visit']) 

1194 

1195 catRecord = zptVisitCatalog.addNew() 

1196 catRecord['id'] = int(rec['detector']) 

1197 catRecord['visit'] = rec['visit'] 

1198 catRecord.setPhotoCalib(photoCalib) 

1199 

1200 # Final output of last exposure catalog 

1201 if returnCatalogs: 

1202 # ensure that the detectors are in sorted order, for fast lookups 

1203 zptVisitCatalog.sort() 

1204 yield (int(lastVisit), zptVisitCatalog) 

1205 

1206 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

1207 """ 

1208 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

1209 and scaling. 

1210 

1211 Parameters 

1212 ---------- 

1213 coefficients: `numpy.array` 

1214 Flattened array of chebyshev coefficients 

1215 xyMax: `list` of length 2 

1216 Maximum x and y of the chebyshev bounding box 

1217 offset: `float`, optional 

1218 Absolute calibration offset. Default is 0.0 

1219 scaling: `float`, optional 

1220 Flat scaling value from fgcmBuildStars. Default is 1.0 

1221 

1222 Returns 

1223 ------- 

1224 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

1225 """ 

1226 

1227 orderPlus1 = int(np.sqrt(coefficients.size)) 

1228 pars = np.zeros((orderPlus1, orderPlus1)) 

1229 

1230 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

1231 lsst.geom.Point2I(*xyMax)) 

1232 

1233 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) 

1234 * (10.**(offset/-2.5))*scaling) 

1235 

1236 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

1237 

1238 return boundedField 

1239 

1240 def _outputAtmospheres(self, dataRefDict, atmCat): 

1241 """ 

1242 Output the atmospheres. 

1243 

1244 Parameters 

1245 ---------- 

1246 dataRefDict : `dict` 

1247 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1248 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1249 dataRef dictionary with keys: 

1250 

1251 ``"fgcmLookUpTable"`` 

1252 dataRef for the FGCM look-up table. 

1253 atmCat : `lsst.afw.table.BaseCatalog` 

1254 FGCM atmosphere parameter catalog from fgcmFitCycleTask. 

1255 

1256 Returns 

1257 ------- 

1258 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

1259 Generator that returns (visit, transmissionCurve) tuples. 

1260 """ 

1261 # First, we need to grab the look-up table and key info 

1262 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1263 

1264 atmosphereTableName = lutCat[0]['tablename'] 

1265 elevation = lutCat[0]['elevation'] 

1266 atmLambda = lutCat[0]['atmLambda'] 

1267 lutCat = None 

1268 

1269 # Make the atmosphere table if possible 

1270 try: 

1271 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

1272 atmTable.loadTable() 

1273 except IOError: 

1274 atmTable = None 

1275 

1276 if atmTable is None: 

1277 # Try to use MODTRAN instead 

1278 try: 

1279 modGen = fgcm.ModtranGenerator(elevation) 

1280 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

1281 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

1282 except (ValueError, IOError) as e: 

1283 raise RuntimeError("FGCM look-up-table generated with modtran, " 

1284 "but modtran not configured to run.") from e 

1285 

1286 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

1287 

1288 for i, visit in enumerate(atmCat['visit']): 

1289 if atmTable is not None: 

1290 # Interpolate the atmosphere table 

1291 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

1292 pwv=atmCat[i]['pwv'], 

1293 o3=atmCat[i]['o3'], 

1294 tau=atmCat[i]['tau'], 

1295 alpha=atmCat[i]['alpha'], 

1296 zenith=zenith[i], 

1297 ctranslamstd=[atmCat[i]['cTrans'], 

1298 atmCat[i]['lamStd']]) 

1299 else: 

1300 # Run modtran 

1301 modAtm = modGen(pmb=atmCat[i]['pmb'], 

1302 pwv=atmCat[i]['pwv'], 

1303 o3=atmCat[i]['o3'], 

1304 tau=atmCat[i]['tau'], 

1305 alpha=atmCat[i]['alpha'], 

1306 zenith=zenith[i], 

1307 lambdaRange=lambdaRange, 

1308 lambdaStep=lambdaStep, 

1309 ctranslamstd=[atmCat[i]['cTrans'], 

1310 atmCat[i]['lamStd']]) 

1311 atmVals = modAtm['COMBINED'] 

1312 

1313 # Now need to create something to persist... 

1314 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

1315 wavelengths=atmLambda, 

1316 throughputAtMin=atmVals[0], 

1317 throughputAtMax=atmVals[-1]) 

1318 

1319 yield (int(visit), curve)