Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34import sys 

35import traceback 

36import copy 

37 

38import numpy as np 

39import healpy as hp 

40import esutil 

41from astropy import units 

42 

43import lsst.pex.config as pexConfig 

44import lsst.pipe.base as pipeBase 

45from lsst.pipe.base import connectionTypes 

46from lsst.afw.image import TransmissionCurve 

47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

48from lsst.meas.algorithms import ReferenceObjectLoader 

49from lsst.pipe.tasks.photoCal import PhotoCalTask 

50import lsst.geom 

51import lsst.afw.image as afwImage 

52import lsst.afw.math as afwMath 

53import lsst.afw.table as afwTable 

54from lsst.meas.algorithms import IndexerRegistry 

55from lsst.meas.algorithms import DatasetConfig 

56from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

57 

58from .utilities import computeApproxPixelAreaFields 

59from .utilities import lookupStaticCalibrations 

60 

61import fgcm 

62 

63__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner'] 

64 

65 

66class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections, 

67 dimensions=("instrument",), 

68 defaultTemplates={"cycleNumber": "0"}): 

69 camera = connectionTypes.PrerequisiteInput( 

70 doc="Camera instrument", 

71 name="camera", 

72 storageClass="Camera", 

73 dimensions=("instrument",), 

74 lookupFunction=lookupStaticCalibrations, 

75 isCalibration=True, 

76 ) 

77 

78 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

79 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

80 "chromatic corrections."), 

81 name="fgcmLookUpTable", 

82 storageClass="Catalog", 

83 dimensions=("instrument",), 

84 deferLoad=True, 

85 ) 

86 

87 fgcmVisitCatalog = connectionTypes.PrerequisiteInput( 

88 doc="Catalog of visit information for fgcm", 

89 name="fgcmVisitCatalog", 

90 storageClass="Catalog", 

91 dimensions=("instrument",), 

92 deferLoad=True, 

93 ) 

94 

95 fgcmStandardStars = connectionTypes.PrerequisiteInput( 

96 doc="Catalog of standard star data from fgcm fit", 

97 name="fgcmStandardStars{cycleNumber}", 

98 storageClass="SimpleCatalog", 

99 dimensions=("instrument",), 

100 deferLoad=True, 

101 ) 

102 

103 fgcmZeropoints = connectionTypes.PrerequisiteInput( 

104 doc="Catalog of zeropoints from fgcm fit", 

105 name="fgcmZeropoints{cycleNumber}", 

106 storageClass="Catalog", 

107 dimensions=("instrument",), 

108 deferLoad=True, 

109 ) 

110 

111 fgcmAtmosphereParameters = connectionTypes.PrerequisiteInput( 

112 doc="Catalog of atmosphere parameters from fgcm fit", 

113 name="fgcmAtmosphereParameters{cycleNumber}", 

114 storageClass="Catalog", 

115 dimensions=("instrument",), 

116 deferLoad=True, 

117 ) 

118 

119 refCat = connectionTypes.PrerequisiteInput( 

120 doc="Reference catalog to use for photometric calibration", 

121 name="cal_ref_cat", 

122 storageClass="SimpleCatalog", 

123 dimensions=("skypix",), 

124 deferLoad=True, 

125 multiple=True, 

126 ) 

127 

128 fgcmBuildStarsTableConfig = connectionTypes.PrerequisiteInput( 

129 doc="Config used to build FGCM input stars", 

130 name="fgcmBuildStarsTable_config", 

131 storageClass="Config", 

132 ) 

133 

134 fgcmPhotoCalib = connectionTypes.Output( 

135 doc="Per-visit photoCalib exposure catalogs produced from fgcm calibration", 

136 name="fgcmPhotoCalibCatalog", 

137 storageClass="ExposureCatalog", 

138 dimensions=("instrument", "visit",), 

139 multiple=True, 

140 ) 

141 

142 fgcmTransmissionAtmosphere = connectionTypes.Output( 

143 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

144 name="transmission_atmosphere_fgcm", 

145 storageClass="TransmissionCurve", 

146 dimensions=("instrument", 

147 "visit",), 

148 multiple=True, 

149 ) 

150 

151 fgcmOffsets = connectionTypes.Output( 

152 doc="Per-band offsets computed from doReferenceCalibration", 

153 name="fgcmReferenceCalibrationOffsets", 

154 storageClass="Catalog", 

155 dimensions=("instrument",), 

156 multiple=False, 

157 ) 

158 

159 def __init__(self, *, config=None): 

160 super().__init__(config=config) 

161 

162 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

163 raise ValueError("cycleNumber must be of integer format") 

164 if config.connections.refCat != config.refObjLoader.ref_dataset_name: 

165 raise ValueError("connections.refCat must be the same as refObjLoader.ref_dataset_name") 

166 

167 if config.doRefcatOutput: 

168 raise ValueError("FgcmOutputProductsTask (Gen3) does not support doRefcatOutput") 

169 

170 if not config.doReferenceCalibration: 

171 self.prerequisiteInputs.remove("refCat") 

172 if not config.doAtmosphereOutput: 

173 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

174 if not config.doZeropointOutput: 

175 self.prerequisiteInputs.remove("fgcmZeropoints") 

176 if not config.doReferenceCalibration: 

177 self.outputs.remove("fgcmOffsets") 

178 

179 

180class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig, 

181 pipelineConnections=FgcmOutputProductsConnections): 

182 """Config for FgcmOutputProductsTask""" 

183 

184 cycleNumber = pexConfig.Field( 

185 doc="Final fit cycle from FGCM fit", 

186 dtype=int, 

187 default=None, 

188 ) 

189 

190 # The following fields refer to calibrating from a reference 

191 # catalog, but in the future this might need to be expanded 

192 doReferenceCalibration = pexConfig.Field( 

193 doc=("Transfer 'absolute' calibration from reference catalog? " 

194 "This afterburner step is unnecessary if reference stars " 

195 "were used in the full fit in FgcmFitCycleTask."), 

196 dtype=bool, 

197 default=False, 

198 ) 

199 doRefcatOutput = pexConfig.Field( 

200 doc="Output standard stars in reference catalog format", 

201 dtype=bool, 

202 default=True, 

203 ) 

204 doAtmosphereOutput = pexConfig.Field( 

205 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

206 dtype=bool, 

207 default=True, 

208 ) 

209 doZeropointOutput = pexConfig.Field( 

210 doc="Output zeropoints in fgcm_photoCalib format", 

211 dtype=bool, 

212 default=True, 

213 ) 

214 doComposeWcsJacobian = pexConfig.Field( 

215 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

216 dtype=bool, 

217 default=True, 

218 ) 

219 doApplyMeanChromaticCorrection = pexConfig.Field( 

220 doc="Apply the mean chromatic correction to the zeropoints?", 

221 dtype=bool, 

222 default=True, 

223 ) 

224 refObjLoader = pexConfig.ConfigurableField( 

225 target=LoadIndexedReferenceObjectsTask, 

226 doc="reference object loader for 'absolute' photometric calibration", 

227 ) 

228 photoCal = pexConfig.ConfigurableField( 

229 target=PhotoCalTask, 

230 doc="task to perform 'absolute' calibration", 

231 ) 

232 referencePixelizationNside = pexConfig.Field( 

233 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

234 dtype=int, 

235 default=64, 

236 ) 

237 referencePixelizationMinStars = pexConfig.Field( 

238 doc=("Minimum number of stars per healpix pixel to select for comparison" 

239 "to the specified reference catalog"), 

240 dtype=int, 

241 default=200, 

242 ) 

243 referenceMinMatch = pexConfig.Field( 

244 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

245 dtype=int, 

246 default=50, 

247 ) 

248 referencePixelizationNPixels = pexConfig.Field( 

249 doc=("Number of healpix pixels to sample to do comparison. " 

250 "Doing too many will take a long time and not yield any more " 

251 "precise results because the final number is the median offset " 

252 "(per band) from the set of pixels."), 

253 dtype=int, 

254 default=100, 

255 ) 

256 datasetConfig = pexConfig.ConfigField( 

257 dtype=DatasetConfig, 

258 doc="Configuration for writing/reading ingested catalog", 

259 ) 

260 

261 def setDefaults(self): 

262 pexConfig.Config.setDefaults(self) 

263 

264 # In order to transfer the "absolute" calibration from a reference 

265 # catalog to the relatively calibrated FGCM standard stars (one number 

266 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

267 # pixels. These basic settings ensure that only well-measured, good stars 

268 # from the source and reference catalogs are used for the matching. 

269 

270 # applyColorTerms needs to be False if doReferenceCalibration is False, 

271 # as is the new default after DM-16702 

272 self.photoCal.applyColorTerms = False 

273 self.photoCal.fluxField = 'instFlux' 

274 self.photoCal.magErrFloor = 0.003 

275 self.photoCal.match.referenceSelection.doSignalToNoise = True 

276 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

277 self.photoCal.match.sourceSelection.doSignalToNoise = True 

278 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

279 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

280 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

281 self.photoCal.match.sourceSelection.doFlags = True 

282 self.photoCal.match.sourceSelection.flags.good = [] 

283 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

284 self.photoCal.match.sourceSelection.doUnresolved = False 

285 self.datasetConfig.ref_dataset_name = 'fgcm_stars' 

286 self.datasetConfig.format_version = 1 

287 

288 def validate(self): 

289 super().validate() 

290 

291 # Force the connections to conform with cycleNumber 

292 self.connections.cycleNumber = str(self.cycleNumber) 

293 

294 

295class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner): 

296 """Subclass of TaskRunner for fgcmOutputProductsTask 

297 

298 fgcmOutputProductsTask.run() takes one argument, the butler, and 

299 does not run on any data in the repository. 

300 This runner does not use any parallelization. 

301 """ 

302 

303 @staticmethod 

304 def getTargetList(parsedCmd): 

305 """ 

306 Return a list with one element, the butler. 

307 """ 

308 return [parsedCmd.butler] 

309 

310 def __call__(self, butler): 

311 """ 

312 Parameters 

313 ---------- 

314 butler: `lsst.daf.persistence.Butler` 

315 

316 Returns 

317 ------- 

318 exitStatus: `list` with `pipeBase.Struct` 

319 exitStatus (0: success; 1: failure) 

320 if self.doReturnResults also 

321 results (`np.array` with absolute zeropoint offsets) 

322 """ 

323 task = self.TaskClass(butler=butler, config=self.config, log=self.log) 

324 

325 exitStatus = 0 

326 if self.doRaise: 

327 results = task.runDataRef(butler) 

328 else: 

329 try: 

330 results = task.runDataRef(butler) 

331 except Exception as e: 

332 exitStatus = 1 

333 task.log.fatal("Failed: %s" % e) 

334 if not isinstance(e, pipeBase.TaskError): 

335 traceback.print_exc(file=sys.stderr) 

336 

337 task.writeMetadata(butler) 

338 

339 if self.doReturnResults: 

340 # The results here are the zeropoint offsets for each band 

341 return [pipeBase.Struct(exitStatus=exitStatus, 

342 results=results)] 

343 else: 

344 return [pipeBase.Struct(exitStatus=exitStatus)] 

345 

346 def run(self, parsedCmd): 

347 """ 

348 Run the task, with no multiprocessing 

349 

350 Parameters 

351 ---------- 

352 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

353 """ 

354 

355 resultList = [] 

356 

357 if self.precall(parsedCmd): 

358 targetList = self.getTargetList(parsedCmd) 

359 # make sure that we only get 1 

360 resultList = self(targetList[0]) 

361 

362 return resultList 

363 

364 

365class FgcmOutputProductsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

366 """ 

367 Output products from FGCM global calibration. 

368 """ 

369 

370 ConfigClass = FgcmOutputProductsConfig 

371 RunnerClass = FgcmOutputProductsRunner 

372 _DefaultName = "fgcmOutputProducts" 

373 

374 def __init__(self, butler=None, **kwargs): 

375 super().__init__(**kwargs) 

376 

377 # no saving of metadata for now 

378 def _getMetadataName(self): 

379 return None 

380 

381 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

382 dataRefDict = {} 

383 dataRefDict['camera'] = butlerQC.get(inputRefs.camera) 

384 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

385 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

386 dataRefDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars) 

387 

388 if self.config.doZeropointOutput: 

389 dataRefDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints) 

390 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

391 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

392 

393 if self.config.doAtmosphereOutput: 

394 dataRefDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters) 

395 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

396 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

397 

398 if self.config.doReferenceCalibration: 

399 refConfig = self.config.refObjLoader 

400 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

401 for ref in inputRefs.refCat], 

402 refCats=butlerQC.get(inputRefs.refCat), 

403 config=refConfig, 

404 log=self.log) 

405 else: 

406 self.refObjLoader = None 

407 

408 dataRefDict['fgcmBuildStarsTableConfig'] = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig) 

409 

410 fgcmBuildStarsConfig = butlerQC.get(inputRefs.fgcmBuildStarsTableConfig) 

411 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

412 

413 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

414 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

415 "in fgcmBuildStarsTask.") 

416 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

417 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

418 

419 struct = self.run(dataRefDict, physicalFilterMap, returnCatalogs=True) 

420 

421 # Output the photoCalib exposure catalogs 

422 if struct.photoCalibCatalogs is not None: 

423 self.log.info("Outputting photoCalib catalogs.") 

424 for visit, expCatalog in struct.photoCalibCatalogs: 

425 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

426 self.log.info("Done outputting photoCalib catalogs.") 

427 

428 # Output the atmospheres 

429 if struct.atmospheres is not None: 

430 self.log.info("Outputting atmosphere transmission files.") 

431 for visit, atm in struct.atmospheres: 

432 butlerQC.put(atm, atmRefDict[visit]) 

433 self.log.info("Done outputting atmosphere files.") 

434 

435 if self.config.doReferenceCalibration: 

436 # Turn offset into simple catalog for persistence if necessary 

437 schema = afwTable.Schema() 

438 schema.addField('offset', type=np.float64, 

439 doc="Post-process calibration offset (mag)") 

440 offsetCat = afwTable.BaseCatalog(schema) 

441 offsetCat.resize(len(struct.offsets)) 

442 offsetCat['offset'][:] = struct.offsets 

443 

444 butlerQC.put(offsetCat, outputRefs.fgcmOffsets) 

445 

446 return 

447 

448 @pipeBase.timeMethod 

449 def runDataRef(self, butler): 

450 """ 

451 Make FGCM output products for use in the stack 

452 

453 Parameters 

454 ---------- 

455 butler: `lsst.daf.persistence.Butler` 

456 cycleNumber: `int` 

457 Final fit cycle number, override config. 

458 

459 Returns 

460 ------- 

461 offsets: `lsst.pipe.base.Struct` 

462 A structure with array of zeropoint offsets 

463 

464 Raises 

465 ------ 

466 RuntimeError: 

467 Raised if any one of the following is true: 

468 

469 - butler cannot find "fgcmBuildStars_config" or 

470 "fgcmBuildStarsTable_config". 

471 - butler cannot find "fgcmFitCycle_config". 

472 - "fgcmFitCycle_config" does not refer to 

473 `self.config.cycleNumber`. 

474 - butler cannot find "fgcmAtmosphereParameters" and 

475 `self.config.doAtmosphereOutput` is `True`. 

476 - butler cannot find "fgcmStandardStars" and 

477 `self.config.doReferenceCalibration` is `True` or 

478 `self.config.doRefcatOutput` is `True`. 

479 - butler cannot find "fgcmZeropoints" and 

480 `self.config.doZeropointOutput` is `True`. 

481 """ 

482 if self.config.doReferenceCalibration: 

483 # We need the ref obj loader to get the flux field 

484 self.makeSubtask("refObjLoader", butler=butler) 

485 

486 # Check to make sure that the fgcmBuildStars config exists, to retrieve 

487 # the visit and ccd dataset tags 

488 if not butler.datasetExists('fgcmBuildStarsTable_config') and \ 

489 not butler.datasetExists('fgcmBuildStars_config'): 

490 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, " 

491 "which is prereq for fgcmOutputProducts") 

492 

493 if butler.datasetExists('fgcmBuildStarsTable_config'): 

494 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config') 

495 else: 

496 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config') 

497 visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

498 ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

499 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

500 

501 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

502 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

503 "in fgcmBuildStarsTask.") 

504 

505 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

506 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

507 

508 # And make sure that the atmosphere was output properly 

509 if (self.config.doAtmosphereOutput 

510 and not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)): 

511 raise RuntimeError(f"Atmosphere parameters are missing for cycle {self.config.cycleNumber}.") 

512 

513 if not butler.datasetExists('fgcmStandardStars', 

514 fgcmcycle=self.config.cycleNumber): 

515 raise RuntimeError("Standard stars are missing for cycle %d." % 

516 (self.config.cycleNumber)) 

517 

518 if (self.config.doZeropointOutput 

519 and (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))): 

520 raise RuntimeError("Zeropoints are missing for cycle %d." % 

521 (self.config.cycleNumber)) 

522 

523 dataRefDict = {} 

524 # This is the _actual_ camera 

525 dataRefDict['camera'] = butler.get('camera') 

526 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

527 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

528 dataRefDict['fgcmStandardStars'] = butler.dataRef('fgcmStandardStars', 

529 fgcmcycle=self.config.cycleNumber) 

530 

531 if self.config.doZeropointOutput: 

532 dataRefDict['fgcmZeropoints'] = butler.dataRef('fgcmZeropoints', 

533 fgcmcycle=self.config.cycleNumber) 

534 if self.config.doAtmosphereOutput: 

535 dataRefDict['fgcmAtmosphereParameters'] = butler.dataRef('fgcmAtmosphereParameters', 

536 fgcmcycle=self.config.cycleNumber) 

537 

538 struct = self.run(dataRefDict, physicalFilterMap, butler=butler, returnCatalogs=False) 

539 

540 if struct.photoCalibs is not None: 

541 self.log.info("Outputting photoCalib files.") 

542 

543 for visit, detector, physicalFilter, photoCalib in struct.photoCalibs: 

544 butler.put(photoCalib, 'fgcm_photoCalib', 

545 dataId={visitDataRefName: visit, 

546 ccdDataRefName: detector, 

547 'filter': physicalFilter}) 

548 

549 self.log.info("Done outputting photoCalib files.") 

550 

551 if struct.atmospheres is not None: 

552 self.log.info("Outputting atmosphere transmission files.") 

553 for visit, atm in struct.atmospheres: 

554 butler.put(atm, "transmission_atmosphere_fgcm", 

555 dataId={visitDataRefName: visit}) 

556 self.log.info("Done outputting atmosphere transmissions.") 

557 

558 return pipeBase.Struct(offsets=struct.offsets) 

559 

560 def run(self, dataRefDict, physicalFilterMap, returnCatalogs=True, butler=None): 

561 """Run the output products task. 

562 

563 Parameters 

564 ---------- 

565 dataRefDict : `dict` 

566 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

567 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

568 dataRef dictionary with keys: 

569 

570 ``"camera"`` 

571 Camera object (`lsst.afw.cameraGeom.Camera`) 

572 ``"fgcmLookUpTable"`` 

573 dataRef for the FGCM look-up table. 

574 ``"fgcmVisitCatalog"`` 

575 dataRef for visit summary catalog. 

576 ``"fgcmStandardStars"`` 

577 dataRef for the output standard star catalog. 

578 ``"fgcmZeropoints"`` 

579 dataRef for the zeropoint data catalog. 

580 ``"fgcmAtmosphereParameters"`` 

581 dataRef for the atmosphere parameter catalog. 

582 ``"fgcmBuildStarsTableConfig"`` 

583 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`. 

584 physicalFilterMap : `dict` 

585 Dictionary of mappings from physical filter to FGCM band. 

586 returnCatalogs : `bool`, optional 

587 Return photoCalibs as per-visit exposure catalogs. 

588 butler : `lsst.daf.persistence.Butler`, optional 

589 Gen2 butler used for reference star outputs 

590 

591 Returns 

592 ------- 

593 retStruct : `lsst.pipe.base.Struct` 

594 Output structure with keys: 

595 

596 offsets : `np.ndarray` 

597 Final reference offsets, per band. 

598 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

599 Generator that returns (visit, transmissionCurve) tuples. 

600 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

601 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

602 (returned if returnCatalogs is False). 

603 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

604 Generator that returns (visit, exposureCatalog) tuples. 

605 (returned if returnCatalogs is True). 

606 """ 

607 stdCat = dataRefDict['fgcmStandardStars'].get() 

608 md = stdCat.getMetadata() 

609 bands = md.getArray('BANDS') 

610 

611 if self.config.doReferenceCalibration: 

612 offsets = self._computeReferenceOffsets(stdCat, bands) 

613 else: 

614 offsets = np.zeros(len(bands)) 

615 

616 # This is Gen2 only, and requires the butler. 

617 if self.config.doRefcatOutput and butler is not None: 

618 self._outputStandardStars(butler, stdCat, offsets, bands, self.config.datasetConfig) 

619 

620 del stdCat 

621 

622 if self.config.doZeropointOutput: 

623 zptCat = dataRefDict['fgcmZeropoints'].get() 

624 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

625 

626 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands, 

627 physicalFilterMap, returnCatalogs=returnCatalogs) 

628 else: 

629 pcgen = None 

630 

631 if self.config.doAtmosphereOutput: 

632 atmCat = dataRefDict['fgcmAtmosphereParameters'].get() 

633 atmgen = self._outputAtmospheres(dataRefDict, atmCat) 

634 else: 

635 atmgen = None 

636 

637 retStruct = pipeBase.Struct(offsets=offsets, 

638 atmospheres=atmgen) 

639 if returnCatalogs: 

640 retStruct.photoCalibCatalogs = pcgen 

641 else: 

642 retStruct.photoCalibs = pcgen 

643 

644 return retStruct 

645 

646 def generateTractOutputProducts(self, dataRefDict, tract, 

647 visitCat, zptCat, atmCat, stdCat, 

648 fgcmBuildStarsConfig, 

649 returnCatalogs=True, 

650 butler=None): 

651 """ 

652 Generate the output products for a given tract, as specified in the config. 

653 

654 This method is here to have an alternate entry-point for 

655 FgcmCalibrateTract. 

656 

657 Parameters 

658 ---------- 

659 dataRefDict : `dict` 

660 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

661 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

662 dataRef dictionary with keys: 

663 

664 ``"camera"`` 

665 Camera object (`lsst.afw.cameraGeom.Camera`) 

666 ``"fgcmLookUpTable"`` 

667 dataRef for the FGCM look-up table. 

668 tract : `int` 

669 Tract number 

670 visitCat : `lsst.afw.table.BaseCatalog` 

671 FGCM visitCat from `FgcmBuildStarsTask` 

672 zptCat : `lsst.afw.table.BaseCatalog` 

673 FGCM zeropoint catalog from `FgcmFitCycleTask` 

674 atmCat : `lsst.afw.table.BaseCatalog` 

675 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

676 stdCat : `lsst.afw.table.SimpleCatalog` 

677 FGCM standard star catalog from `FgcmFitCycleTask` 

678 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig` 

679 Configuration object from `FgcmBuildStarsTask` 

680 returnCatalogs : `bool`, optional 

681 Return photoCalibs as per-visit exposure catalogs. 

682 butler: `lsst.daf.persistence.Butler`, optional 

683 Gen2 butler used for reference star outputs 

684 

685 Returns 

686 ------- 

687 retStruct : `lsst.pipe.base.Struct` 

688 Output structure with keys: 

689 

690 offsets : `np.ndarray` 

691 Final reference offsets, per band. 

692 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

693 Generator that returns (visit, transmissionCurve) tuples. 

694 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

695 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

696 (returned if returnCatalogs is False). 

697 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

698 Generator that returns (visit, exposureCatalog) tuples. 

699 (returned if returnCatalogs is True). 

700 """ 

701 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

702 

703 md = stdCat.getMetadata() 

704 bands = md.getArray('BANDS') 

705 

706 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

707 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

708 "in fgcmBuildStarsTask.") 

709 

710 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

711 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

712 

713 if self.config.doReferenceCalibration: 

714 offsets = self._computeReferenceOffsets(stdCat, bands) 

715 else: 

716 offsets = np.zeros(len(bands)) 

717 

718 if self.config.doRefcatOutput and butler is not None: 

719 # Create a special config that has the tract number in it 

720 datasetConfig = copy.copy(self.config.datasetConfig) 

721 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name, 

722 tract) 

723 self._outputStandardStars(butler, stdCat, offsets, bands, datasetConfig) 

724 

725 if self.config.doZeropointOutput: 

726 pcgen = self._outputZeropoints(dataRefDict['camera'], zptCat, visitCat, offsets, bands, 

727 physicalFilterMap, returnCatalogs=returnCatalogs) 

728 else: 

729 pcgen = None 

730 

731 if self.config.doAtmosphereOutput: 

732 atmgen = self._outputAtmospheres(dataRefDict, atmCat) 

733 else: 

734 atmgen = None 

735 

736 retStruct = pipeBase.Struct(offsets=offsets, 

737 atmospheres=atmgen) 

738 if returnCatalogs: 

739 retStruct.photoCalibCatalogs = pcgen 

740 else: 

741 retStruct.photoCalibs = pcgen 

742 

743 return retStruct 

744 

745 def _computeReferenceOffsets(self, stdCat, bands): 

746 """ 

747 Compute offsets relative to a reference catalog. 

748 

749 This method splits the star catalog into healpix pixels 

750 and computes the calibration transfer for a sample of 

751 these pixels to approximate the 'absolute' calibration 

752 values (on for each band) to apply to transfer the 

753 absolute scale. 

754 

755 Parameters 

756 ---------- 

757 stdCat : `lsst.afw.table.SimpleCatalog` 

758 FGCM standard stars 

759 bands : `list` [`str`] 

760 List of band names from FGCM output 

761 Returns 

762 ------- 

763 offsets : `numpy.array` of floats 

764 Per band zeropoint offsets 

765 """ 

766 

767 # Only use stars that are observed in all the bands that were actually used 

768 # This will ensure that we use the same healpix pixels for the absolute 

769 # calibration of each band. 

770 minObs = stdCat['ngood'].min(axis=1) 

771 

772 goodStars = (minObs >= 1) 

773 stdCat = stdCat[goodStars] 

774 

775 self.log.info("Found %d stars with at least 1 good observation in each band" % 

776 (len(stdCat))) 

777 

778 # We have to make a table for each pixel with flux/fluxErr 

779 # This is a temporary table generated for input to the photoCal task. 

780 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

781 # have had chromatic corrections applied to get to the standard system 

782 # specified by the atmosphere/instrumental parameters), nor are they 

783 # in Jansky (since they don't have a proper absolute calibration: the overall 

784 # zeropoint is estimated from the telescope size, etc.) 

785 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

786 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

787 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

788 doc="instrumental flux (counts)") 

789 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

790 doc="instrumental flux error (counts)") 

791 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

792 type='Flag', 

793 doc="bad flag") 

794 

795 # Split up the stars 

796 # Note that there is an assumption here that the ra/dec coords stored 

797 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

798 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

799 theta = np.pi/2. - stdCat['coord_dec'] 

800 phi = stdCat['coord_ra'] 

801 

802 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) 

803 h, rev = esutil.stat.histogram(ipring, rev=True) 

804 

805 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

806 

807 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

808 (gdpix.size, 

809 self.config.referencePixelizationNside, 

810 self.config.referencePixelizationMinStars)) 

811 

812 if gdpix.size < self.config.referencePixelizationNPixels: 

813 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

814 (gdpix.size, self.config.referencePixelizationNPixels)) 

815 else: 

816 # Sample out the pixels we want to use 

817 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

818 

819 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

820 ('nstar', 'i4', len(bands)), 

821 ('nmatch', 'i4', len(bands)), 

822 ('zp', 'f4', len(bands)), 

823 ('zpErr', 'f4', len(bands))]) 

824 results['hpix'] = ipring[rev[rev[gdpix]]] 

825 

826 # We need a boolean index to deal with catalogs... 

827 selected = np.zeros(len(stdCat), dtype=np.bool) 

828 

829 refFluxFields = [None]*len(bands) 

830 

831 for p, pix in enumerate(gdpix): 

832 i1a = rev[rev[pix]: rev[pix + 1]] 

833 

834 # the stdCat afwTable can only be indexed with boolean arrays, 

835 # and not numpy index arrays (see DM-16497). This little trick 

836 # converts the index array into a boolean array 

837 selected[:] = False 

838 selected[i1a] = True 

839 

840 for b, band in enumerate(bands): 

841 

842 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat, 

843 selected, refFluxFields) 

844 results['nstar'][p, b] = len(i1a) 

845 results['nmatch'][p, b] = len(struct.arrays.refMag) 

846 results['zp'][p, b] = struct.zp 

847 results['zpErr'][p, b] = struct.sigma 

848 

849 # And compute the summary statistics 

850 offsets = np.zeros(len(bands)) 

851 

852 for b, band in enumerate(bands): 

853 # make configurable 

854 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch) 

855 offsets[b] = np.median(results['zp'][ok, b]) 

856 # use median absolute deviation to estimate Normal sigma 

857 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

858 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b])) 

859 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" % 

860 (band, offsets[b], madSigma)) 

861 

862 return offsets 

863 

864 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

865 b, band, stdCat, selected, refFluxFields): 

866 """ 

867 Compute the zeropoint offset between the fgcm stdCat and the reference 

868 stars for one pixel in one band 

869 

870 Parameters 

871 ---------- 

872 sourceMapper: `lsst.afw.table.SchemaMapper` 

873 Mapper to go from stdCat to calibratable catalog 

874 badStarKey: `lsst.afw.table.Key` 

875 Key for the field with bad stars 

876 b: `int` 

877 Index of the band in the star catalog 

878 band: `str` 

879 Name of band for reference catalog 

880 stdCat: `lsst.afw.table.SimpleCatalog` 

881 FGCM standard stars 

882 selected: `numpy.array(dtype=np.bool)` 

883 Boolean array of which stars are in the pixel 

884 refFluxFields: `list` 

885 List of names of flux fields for reference catalog 

886 """ 

887 

888 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

889 sourceCat.reserve(selected.sum()) 

890 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

891 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5)) 

892 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] 

893 * sourceCat['instFlux']) 

894 # Make sure we only use stars that have valid measurements 

895 # (This is perhaps redundant with requirements above that the 

896 # stars be observed in all bands, but it can't hurt) 

897 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0) 

898 for rec in sourceCat[badStar]: 

899 rec.set(badStarKey, True) 

900 

901 exposure = afwImage.ExposureF() 

902 exposure.setFilter(afwImage.Filter(band)) 

903 

904 if refFluxFields[b] is None: 

905 # Need to find the flux field in the reference catalog 

906 # to work around limitations of DirectMatch in PhotoCal 

907 ctr = stdCat[0].getCoord() 

908 rad = 0.05*lsst.geom.degrees 

909 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band) 

910 refFluxFields[b] = refDataTest.fluxField 

911 

912 # Make a copy of the config so that we can modify it 

913 calConfig = copy.copy(self.config.photoCal.value) 

914 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b] 

915 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err' 

916 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

917 config=calConfig, 

918 schema=sourceCat.getSchema()) 

919 

920 struct = calTask.run(exposure, sourceCat) 

921 

922 return struct 

923 

924 def _outputStandardStars(self, butler, stdCat, offsets, bands, datasetConfig): 

925 """ 

926 Output standard stars in indexed reference catalog format. 

927 This is not currently supported in Gen3. 

928 

929 Parameters 

930 ---------- 

931 butler : `lsst.daf.persistence.Butler` 

932 stdCat : `lsst.afw.table.SimpleCatalog` 

933 FGCM standard star catalog from fgcmFitCycleTask 

934 offsets : `numpy.array` of floats 

935 Per band zeropoint offsets 

936 bands : `list` [`str`] 

937 List of band names from FGCM output 

938 datasetConfig : `lsst.meas.algorithms.DatasetConfig` 

939 Config for reference dataset 

940 """ 

941 

942 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name)) 

943 

944 indexer = IndexerRegistry[self.config.datasetConfig.indexer.name]( 

945 self.config.datasetConfig.indexer.active) 

946 

947 # We determine the conversion from the native units (typically radians) to 

948 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

949 # numpy arrays rather than Angles, which would we approximately 600x slower. 

950 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords 

951 # (as Angles) for input 

952 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra']) 

953 indices = np.array(indexer.indexPoints(stdCat['coord_ra']*conv, 

954 stdCat['coord_dec']*conv)) 

955 

956 formattedCat = self._formatCatalog(stdCat, offsets, bands) 

957 

958 # Write the master schema 

959 dataId = indexer.makeDataId('master_schema', 

960 datasetConfig.ref_dataset_name) 

961 masterCat = afwTable.SimpleCatalog(formattedCat.schema) 

962 addRefCatMetadata(masterCat) 

963 butler.put(masterCat, 'ref_cat', dataId=dataId) 

964 

965 # Break up the pixels using a histogram 

966 h, rev = esutil.stat.histogram(indices, rev=True) 

967 gd, = np.where(h > 0) 

968 selected = np.zeros(len(formattedCat), dtype=np.bool) 

969 for i in gd: 

970 i1a = rev[rev[i]: rev[i + 1]] 

971 

972 # the formattedCat afwTable can only be indexed with boolean arrays, 

973 # and not numpy index arrays (see DM-16497). This little trick 

974 # converts the index array into a boolean array 

975 selected[:] = False 

976 selected[i1a] = True 

977 

978 # Write the individual pixel 

979 dataId = indexer.makeDataId(indices[i1a[0]], 

980 datasetConfig.ref_dataset_name) 

981 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId) 

982 

983 # And save the dataset configuration 

984 dataId = indexer.makeDataId(None, datasetConfig.ref_dataset_name) 

985 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId) 

986 

987 self.log.info("Done outputting standard stars.") 

988 

989 def _formatCatalog(self, fgcmStarCat, offsets, bands): 

990 """ 

991 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

992 

993 Parameters 

994 ---------- 

995 fgcmStarCat : `lsst.afw.Table.SimpleCatalog` 

996 SimpleCatalog as output by fgcmcal 

997 offsets : `list` with len(self.bands) entries 

998 Zeropoint offsets to apply 

999 bands : `list` [`str`] 

1000 List of band names from FGCM output 

1001 

1002 Returns 

1003 ------- 

1004 formattedCat: `lsst.afw.table.SimpleCatalog` 

1005 SimpleCatalog suitable for using as a reference catalog 

1006 """ 

1007 

1008 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

1009 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands, 

1010 addCentroid=False, 

1011 addIsResolved=True, 

1012 coordErrDim=0) 

1013 sourceMapper.addMinimalSchema(minSchema) 

1014 for band in bands: 

1015 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

1016 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

1017 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

1018 

1019 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

1020 formattedCat.reserve(len(fgcmStarCat)) 

1021 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

1022 

1023 # Note that we don't have to set `resolved` because the default is False 

1024 

1025 for b, band in enumerate(bands): 

1026 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

1027 # We want fluxes in nJy from calibrated AB magnitudes 

1028 # (after applying offset). Updated after RFC-549 and RFC-575. 

1029 flux = (mag*units.ABmag).to_value(units.nJy) 

1030 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

1031 

1032 formattedCat['%s_flux' % (band)][:] = flux 

1033 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

1034 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

1035 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

1036 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

1037 

1038 addRefCatMetadata(formattedCat) 

1039 

1040 return formattedCat 

1041 

1042 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands, 

1043 physicalFilterMap, returnCatalogs=True, 

1044 tract=None): 

1045 """Output the zeropoints in fgcm_photoCalib format. 

1046 

1047 Parameters 

1048 ---------- 

1049 camera : `lsst.afw.cameraGeom.Camera` 

1050 Camera from the butler. 

1051 zptCat : `lsst.afw.table.BaseCatalog` 

1052 FGCM zeropoint catalog from `FgcmFitCycleTask`. 

1053 visitCat : `lsst.afw.table.BaseCatalog` 

1054 FGCM visitCat from `FgcmBuildStarsTask`. 

1055 offsets : `numpy.array` 

1056 Float array of absolute calibration offsets, one for each filter. 

1057 bands : `list` [`str`] 

1058 List of band names from FGCM output. 

1059 physicalFilterMap : `dict` 

1060 Dictionary of mappings from physical filter to FGCM band. 

1061 returnCatalogs : `bool`, optional 

1062 Return photoCalibs as per-visit exposure catalogs. 

1063 tract: `int`, optional 

1064 Tract number to output. Default is None (global calibration) 

1065 

1066 Returns 

1067 ------- 

1068 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

1069 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

1070 (returned if returnCatalogs is False). 

1071 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

1072 Generator that returns (visit, exposureCatalog) tuples. 

1073 (returned if returnCatalogs is True). 

1074 """ 

1075 # Select visit/ccds where we have a calibration 

1076 # This includes ccds where we were able to interpolate from neighboring 

1077 # ccds. 

1078 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

1079 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) 

1080 & (zptCat['fgcmZptVar'] > 0.0)) 

1081 

1082 # Log warnings for any visit which has no valid zeropoints 

1083 badVisits = np.unique(zptCat['visit'][~selected]) 

1084 goodVisits = np.unique(zptCat['visit'][selected]) 

1085 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

1086 for allBadVisit in allBadVisits: 

1087 self.log.warn(f'No suitable photoCalib for visit {allBadVisit}') 

1088 

1089 # Get a mapping from filtername to the offsets 

1090 offsetMapping = {} 

1091 for f in physicalFilterMap: 

1092 # Not every filter in the map will necesarily have a band. 

1093 if physicalFilterMap[f] in bands: 

1094 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])] 

1095 

1096 # Get a mapping from "ccd" to the ccd index used for the scaling 

1097 ccdMapping = {} 

1098 for ccdIndex, detector in enumerate(camera): 

1099 ccdMapping[detector.getId()] = ccdIndex 

1100 

1101 # And a mapping to get the flat-field scaling values 

1102 scalingMapping = {} 

1103 for rec in visitCat: 

1104 scalingMapping[rec['visit']] = rec['scaling'] 

1105 

1106 if self.config.doComposeWcsJacobian: 

1107 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

1108 

1109 # The zptCat is sorted by visit, which is useful 

1110 lastVisit = -1 

1111 zptCounter = 0 

1112 zptVisitCatalog = None 

1113 for rec in zptCat[selected]: 

1114 

1115 # Retrieve overall scaling 

1116 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]] 

1117 

1118 # The postCalibrationOffset describe any zeropoint offsets 

1119 # to apply after the fgcm calibration. The first part comes 

1120 # from the reference catalog match (used in testing). The 

1121 # second part comes from the mean chromatic correction 

1122 # (if configured). 

1123 postCalibrationOffset = offsetMapping[rec['filtername']] 

1124 if self.config.doApplyMeanChromaticCorrection: 

1125 postCalibrationOffset += rec['fgcmDeltaChrom'] 

1126 

1127 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

1128 rec['fgcmfZptChebXyMax']) 

1129 # Convert from FGCM AB to nJy 

1130 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

1131 rec['fgcmfZptChebXyMax'], 

1132 offset=postCalibrationOffset, 

1133 scaling=scaling) 

1134 

1135 if self.config.doComposeWcsJacobian: 

1136 

1137 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']], 

1138 fgcmSuperStarField, 

1139 fgcmZptField]) 

1140 else: 

1141 # The photoCalib is just the product of the fgcmSuperStarField and the 

1142 # fgcmZptField 

1143 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

1144 

1145 # The "mean" calibration will be set to the center of the ccd for reference 

1146 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

1147 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

1148 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

1149 calibrationErr=calibErr, 

1150 calibration=fgcmField, 

1151 isConstant=False) 

1152 

1153 if not returnCatalogs: 

1154 # Return individual photoCalibs 

1155 yield (int(rec['visit']), int(rec['detector']), rec['filtername'], photoCalib) 

1156 else: 

1157 # Return full per-visit exposure catalogs 

1158 if rec['visit'] != lastVisit: 

1159 # This is a new visit. If the last visit was not -1, yield 

1160 # the ExposureCatalog 

1161 if lastVisit > -1: 

1162 yield (int(lastVisit), zptVisitCatalog) 

1163 else: 

1164 # We need to create a new schema 

1165 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema() 

1166 zptExpCatSchema.addField('visit', type='I', doc='Visit number') 

1167 zptExpCatSchema.addField('detector_id', type='I', doc='Detector number') 

1168 

1169 # And start a new one 

1170 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema) 

1171 zptVisitCatalog.resize(len(camera)) 

1172 zptVisitCatalog['visit'] = rec['visit'] 

1173 # By default all records will not resolve to a valid detector. 

1174 zptVisitCatalog['detector_id'] = -1 

1175 

1176 # Reset the counter 

1177 zptCounter = 0 

1178 

1179 lastVisit = int(rec['visit']) 

1180 

1181 zptVisitCatalog[zptCounter].setPhotoCalib(photoCalib) 

1182 zptVisitCatalog[zptCounter]['detector_id'] = int(rec['detector']) 

1183 

1184 zptCounter += 1 

1185 

1186 # Final output of last exposure catalog 

1187 if returnCatalogs: 

1188 yield (int(lastVisit), zptVisitCatalog) 

1189 

1190 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

1191 """ 

1192 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

1193 and scaling. 

1194 

1195 Parameters 

1196 ---------- 

1197 coefficients: `numpy.array` 

1198 Flattened array of chebyshev coefficients 

1199 xyMax: `list` of length 2 

1200 Maximum x and y of the chebyshev bounding box 

1201 offset: `float`, optional 

1202 Absolute calibration offset. Default is 0.0 

1203 scaling: `float`, optional 

1204 Flat scaling value from fgcmBuildStars. Default is 1.0 

1205 

1206 Returns 

1207 ------- 

1208 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

1209 """ 

1210 

1211 orderPlus1 = int(np.sqrt(coefficients.size)) 

1212 pars = np.zeros((orderPlus1, orderPlus1)) 

1213 

1214 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

1215 lsst.geom.Point2I(*xyMax)) 

1216 

1217 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) 

1218 * (10.**(offset/-2.5))*scaling) 

1219 

1220 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

1221 

1222 return boundedField 

1223 

1224 def _outputAtmospheres(self, dataRefDict, atmCat): 

1225 """ 

1226 Output the atmospheres. 

1227 

1228 Parameters 

1229 ---------- 

1230 dataRefDict : `dict` 

1231 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1232 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1233 dataRef dictionary with keys: 

1234 

1235 ``"fgcmLookUpTable"`` 

1236 dataRef for the FGCM look-up table. 

1237 atmCat : `lsst.afw.table.BaseCatalog` 

1238 FGCM atmosphere parameter catalog from fgcmFitCycleTask. 

1239 

1240 Returns 

1241 ------- 

1242 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

1243 Generator that returns (visit, transmissionCurve) tuples. 

1244 """ 

1245 # First, we need to grab the look-up table and key info 

1246 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1247 

1248 atmosphereTableName = lutCat[0]['tablename'] 

1249 elevation = lutCat[0]['elevation'] 

1250 atmLambda = lutCat[0]['atmLambda'] 

1251 lutCat = None 

1252 

1253 # Make the atmosphere table if possible 

1254 try: 

1255 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

1256 atmTable.loadTable() 

1257 except IOError: 

1258 atmTable = None 

1259 

1260 if atmTable is None: 

1261 # Try to use MODTRAN instead 

1262 try: 

1263 modGen = fgcm.ModtranGenerator(elevation) 

1264 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

1265 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

1266 except (ValueError, IOError) as e: 

1267 raise RuntimeError("FGCM look-up-table generated with modtran, " 

1268 "but modtran not configured to run.") from e 

1269 

1270 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

1271 

1272 for i, visit in enumerate(atmCat['visit']): 

1273 if atmTable is not None: 

1274 # Interpolate the atmosphere table 

1275 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

1276 pwv=atmCat[i]['pwv'], 

1277 o3=atmCat[i]['o3'], 

1278 tau=atmCat[i]['tau'], 

1279 alpha=atmCat[i]['alpha'], 

1280 zenith=zenith[i], 

1281 ctranslamstd=[atmCat[i]['cTrans'], 

1282 atmCat[i]['lamStd']]) 

1283 else: 

1284 # Run modtran 

1285 modAtm = modGen(pmb=atmCat[i]['pmb'], 

1286 pwv=atmCat[i]['pwv'], 

1287 o3=atmCat[i]['o3'], 

1288 tau=atmCat[i]['tau'], 

1289 alpha=atmCat[i]['alpha'], 

1290 zenith=zenith[i], 

1291 lambdaRange=lambdaRange, 

1292 lambdaStep=lambdaStep, 

1293 ctranslamstd=[atmCat[i]['cTrans'], 

1294 atmCat[i]['lamStd']]) 

1295 atmVals = modAtm['COMBINED'] 

1296 

1297 # Now need to create something to persist... 

1298 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

1299 wavelengths=atmLambda, 

1300 throughputAtMin=atmVals[0], 

1301 throughputAtMax=atmVals[-1]) 

1302 

1303 yield (int(visit), curve)