Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py: 15%

314 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-25 12:04 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34import copy 

35 

36import numpy as np 

37import hpgeom as hpg 

38import esutil 

39from astropy import units 

40 

41import lsst.daf.base as dafBase 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45from lsst.afw.image import TransmissionCurve 

46from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

47from lsst.pipe.tasks.photoCal import PhotoCalTask 

48import lsst.geom 

49import lsst.afw.image as afwImage 

50import lsst.afw.math as afwMath 

51import lsst.afw.table as afwTable 

52 

53from .utilities import computeApproxPixelAreaFields 

54from .utilities import lookupStaticCalibrations 

55from .utilities import FGCM_ILLEGAL_VALUE 

56 

57import fgcm 

58 

59__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask'] 

60 

61 

62class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections, 

63 dimensions=("instrument",), 

64 defaultTemplates={"cycleNumber": "0"}): 

65 camera = connectionTypes.PrerequisiteInput( 

66 doc="Camera instrument", 

67 name="camera", 

68 storageClass="Camera", 

69 dimensions=("instrument",), 

70 lookupFunction=lookupStaticCalibrations, 

71 isCalibration=True, 

72 ) 

73 

74 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

75 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

76 "chromatic corrections."), 

77 name="fgcmLookUpTable", 

78 storageClass="Catalog", 

79 dimensions=("instrument",), 

80 deferLoad=True, 

81 ) 

82 

83 fgcmVisitCatalog = connectionTypes.Input( 

84 doc="Catalog of visit information for fgcm", 

85 name="fgcmVisitCatalog", 

86 storageClass="Catalog", 

87 dimensions=("instrument",), 

88 deferLoad=True, 

89 ) 

90 

91 fgcmStandardStars = connectionTypes.Input( 

92 doc="Catalog of standard star data from fgcm fit", 

93 name="fgcmStandardStars{cycleNumber}", 

94 storageClass="SimpleCatalog", 

95 dimensions=("instrument",), 

96 deferLoad=True, 

97 ) 

98 

99 fgcmZeropoints = connectionTypes.Input( 

100 doc="Catalog of zeropoints from fgcm fit", 

101 name="fgcmZeropoints{cycleNumber}", 

102 storageClass="Catalog", 

103 dimensions=("instrument",), 

104 deferLoad=True, 

105 ) 

106 

107 fgcmAtmosphereParameters = connectionTypes.Input( 

108 doc="Catalog of atmosphere parameters from fgcm fit", 

109 name="fgcmAtmosphereParameters{cycleNumber}", 

110 storageClass="Catalog", 

111 dimensions=("instrument",), 

112 deferLoad=True, 

113 ) 

114 

115 refCat = connectionTypes.PrerequisiteInput( 

116 doc="Reference catalog to use for photometric calibration", 

117 name="cal_ref_cat", 

118 storageClass="SimpleCatalog", 

119 dimensions=("skypix",), 

120 deferLoad=True, 

121 multiple=True, 

122 ) 

123 

124 fgcmPhotoCalib = connectionTypes.Output( 

125 doc=("Per-visit photometric calibrations derived from fgcm calibration. " 

126 "These catalogs use detector id for the id and are sorted for " 

127 "fast lookups of a detector."), 

128 name="fgcmPhotoCalibCatalog", 

129 storageClass="ExposureCatalog", 

130 dimensions=("instrument", "visit",), 

131 multiple=True, 

132 ) 

133 

134 fgcmTransmissionAtmosphere = connectionTypes.Output( 

135 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

136 name="transmission_atmosphere_fgcm", 

137 storageClass="TransmissionCurve", 

138 dimensions=("instrument", 

139 "visit",), 

140 multiple=True, 

141 ) 

142 

143 fgcmOffsets = connectionTypes.Output( 

144 doc="Per-band offsets computed from doReferenceCalibration", 

145 name="fgcmReferenceCalibrationOffsets", 

146 storageClass="Catalog", 

147 dimensions=("instrument",), 

148 multiple=False, 

149 ) 

150 

151 def __init__(self, *, config=None): 

152 super().__init__(config=config) 

153 

154 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

155 raise ValueError("cycleNumber must be of integer format") 

156 

157 if not config.doReferenceCalibration: 

158 self.prerequisiteInputs.remove("refCat") 

159 if not config.doAtmosphereOutput: 

160 self.inputs.remove("fgcmAtmosphereParameters") 

161 if not config.doZeropointOutput: 

162 self.inputs.remove("fgcmZeropoints") 

163 if not config.doReferenceCalibration: 

164 self.outputs.remove("fgcmOffsets") 

165 

166 def getSpatialBoundsConnections(self): 

167 return ("fgcmPhotoCalib",) 

168 

169 

170class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig, 

171 pipelineConnections=FgcmOutputProductsConnections): 

172 """Config for FgcmOutputProductsTask""" 

173 

174 physicalFilterMap = pexConfig.DictField( 

175 doc="Mapping from 'physicalFilter' to band.", 

176 keytype=str, 

177 itemtype=str, 

178 default={}, 

179 ) 

180 # The following fields refer to calibrating from a reference 

181 # catalog, but in the future this might need to be expanded 

182 doReferenceCalibration = pexConfig.Field( 

183 doc=("Transfer 'absolute' calibration from reference catalog? " 

184 "This afterburner step is unnecessary if reference stars " 

185 "were used in the full fit in FgcmFitCycleTask."), 

186 dtype=bool, 

187 default=False, 

188 ) 

189 doAtmosphereOutput = pexConfig.Field( 

190 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

191 dtype=bool, 

192 default=True, 

193 ) 

194 doZeropointOutput = pexConfig.Field( 

195 doc="Output zeropoints in fgcm_photoCalib format", 

196 dtype=bool, 

197 default=True, 

198 ) 

199 doComposeWcsJacobian = pexConfig.Field( 

200 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

201 dtype=bool, 

202 default=True, 

203 ) 

204 doApplyMeanChromaticCorrection = pexConfig.Field( 

205 doc="Apply the mean chromatic correction to the zeropoints?", 

206 dtype=bool, 

207 default=True, 

208 ) 

209 photoCal = pexConfig.ConfigurableField( 

210 target=PhotoCalTask, 

211 doc="task to perform 'absolute' calibration", 

212 ) 

213 referencePixelizationNside = pexConfig.Field( 

214 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

215 dtype=int, 

216 default=64, 

217 ) 

218 referencePixelizationMinStars = pexConfig.Field( 

219 doc=("Minimum number of stars per healpix pixel to select for comparison" 

220 "to the specified reference catalog"), 

221 dtype=int, 

222 default=200, 

223 ) 

224 referenceMinMatch = pexConfig.Field( 

225 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

226 dtype=int, 

227 default=50, 

228 ) 

229 referencePixelizationNPixels = pexConfig.Field( 

230 doc=("Number of healpix pixels to sample to do comparison. " 

231 "Doing too many will take a long time and not yield any more " 

232 "precise results because the final number is the median offset " 

233 "(per band) from the set of pixels."), 

234 dtype=int, 

235 default=100, 

236 ) 

237 

238 def setDefaults(self): 

239 pexConfig.Config.setDefaults(self) 

240 

241 # In order to transfer the "absolute" calibration from a reference 

242 # catalog to the relatively calibrated FGCM standard stars (one number 

243 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

244 # pixels. These basic settings ensure that only well-measured, good stars 

245 # from the source and reference catalogs are used for the matching. 

246 

247 # applyColorTerms needs to be False if doReferenceCalibration is False, 

248 # as is the new default after DM-16702 

249 self.photoCal.applyColorTerms = False 

250 self.photoCal.fluxField = 'instFlux' 

251 self.photoCal.magErrFloor = 0.003 

252 self.photoCal.match.referenceSelection.doSignalToNoise = True 

253 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

254 self.photoCal.match.sourceSelection.doSignalToNoise = True 

255 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

256 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

257 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

258 self.photoCal.match.sourceSelection.doFlags = True 

259 self.photoCal.match.sourceSelection.flags.good = [] 

260 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

261 self.photoCal.match.sourceSelection.doUnresolved = False 

262 self.photoCal.match.sourceSelection.doRequirePrimary = False 

263 

264 

265class FgcmOutputProductsTask(pipeBase.PipelineTask): 

266 """ 

267 Output products from FGCM global calibration. 

268 """ 

269 

270 ConfigClass = FgcmOutputProductsConfig 

271 _DefaultName = "fgcmOutputProducts" 

272 

273 def __init__(self, **kwargs): 

274 super().__init__(**kwargs) 

275 

276 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

277 handleDict = {} 

278 handleDict['camera'] = butlerQC.get(inputRefs.camera) 

279 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

280 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

281 handleDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars) 

282 

283 if self.config.doZeropointOutput: 

284 handleDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints) 

285 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

286 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

287 

288 if self.config.doAtmosphereOutput: 

289 handleDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters) 

290 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

291 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

292 

293 if self.config.doReferenceCalibration: 

294 refConfig = LoadReferenceObjectsConfig() 

295 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

296 for ref in inputRefs.refCat], 

297 refCats=butlerQC.get(inputRefs.refCat), 

298 name=self.config.connections.refCat, 

299 log=self.log, 

300 config=refConfig) 

301 else: 

302 self.refObjLoader = None 

303 

304 struct = self.run(handleDict, self.config.physicalFilterMap) 

305 

306 # Output the photoCalib exposure catalogs 

307 if struct.photoCalibCatalogs is not None: 

308 self.log.info("Outputting photoCalib catalogs.") 

309 for visit, expCatalog in struct.photoCalibCatalogs: 

310 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

311 self.log.info("Done outputting photoCalib catalogs.") 

312 

313 # Output the atmospheres 

314 if struct.atmospheres is not None: 

315 self.log.info("Outputting atmosphere transmission files.") 

316 for visit, atm in struct.atmospheres: 

317 butlerQC.put(atm, atmRefDict[visit]) 

318 self.log.info("Done outputting atmosphere files.") 

319 

320 if self.config.doReferenceCalibration: 

321 # Turn offset into simple catalog for persistence if necessary 

322 schema = afwTable.Schema() 

323 schema.addField('offset', type=np.float64, 

324 doc="Post-process calibration offset (mag)") 

325 offsetCat = afwTable.BaseCatalog(schema) 

326 offsetCat.resize(len(struct.offsets)) 

327 offsetCat['offset'][:] = struct.offsets 

328 

329 butlerQC.put(offsetCat, outputRefs.fgcmOffsets) 

330 

331 return 

332 

333 def run(self, handleDict, physicalFilterMap): 

334 """Run the output products task. 

335 

336 Parameters 

337 ---------- 

338 handleDict : `dict` 

339 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

340 handle dictionary with keys: 

341 

342 ``"camera"`` 

343 Camera object (`lsst.afw.cameraGeom.Camera`) 

344 ``"fgcmLookUpTable"`` 

345 handle for the FGCM look-up table. 

346 ``"fgcmVisitCatalog"`` 

347 handle for visit summary catalog. 

348 ``"fgcmStandardStars"`` 

349 handle for the output standard star catalog. 

350 ``"fgcmZeropoints"`` 

351 handle for the zeropoint data catalog. 

352 ``"fgcmAtmosphereParameters"`` 

353 handle for the atmosphere parameter catalog. 

354 ``"fgcmBuildStarsTableConfig"`` 

355 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`. 

356 physicalFilterMap : `dict` 

357 Dictionary of mappings from physical filter to FGCM band. 

358 

359 Returns 

360 ------- 

361 retStruct : `lsst.pipe.base.Struct` 

362 Output structure with keys: 

363 

364 offsets : `np.ndarray` 

365 Final reference offsets, per band. 

366 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

367 Generator that returns (visit, transmissionCurve) tuples. 

368 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

369 Generator that returns (visit, exposureCatalog) tuples. 

370 """ 

371 stdCat = handleDict['fgcmStandardStars'].get() 

372 md = stdCat.getMetadata() 

373 bands = md.getArray('BANDS') 

374 

375 if self.config.doReferenceCalibration: 

376 lutCat = handleDict['fgcmLookUpTable'].get() 

377 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands) 

378 else: 

379 offsets = np.zeros(len(bands)) 

380 

381 del stdCat 

382 

383 if self.config.doZeropointOutput: 

384 zptCat = handleDict['fgcmZeropoints'].get() 

385 visitCat = handleDict['fgcmVisitCatalog'].get() 

386 

387 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands, 

388 physicalFilterMap) 

389 else: 

390 pcgen = None 

391 

392 if self.config.doAtmosphereOutput: 

393 atmCat = handleDict['fgcmAtmosphereParameters'].get() 

394 atmgen = self._outputAtmospheres(handleDict, atmCat) 

395 else: 

396 atmgen = None 

397 

398 retStruct = pipeBase.Struct(offsets=offsets, 

399 atmospheres=atmgen) 

400 retStruct.photoCalibCatalogs = pcgen 

401 

402 return retStruct 

403 

404 def generateTractOutputProducts(self, handleDict, tract, 

405 visitCat, zptCat, atmCat, stdCat, 

406 fgcmBuildStarsConfig): 

407 """ 

408 Generate the output products for a given tract, as specified in the config. 

409 

410 This method is here to have an alternate entry-point for 

411 FgcmCalibrateTract. 

412 

413 Parameters 

414 ---------- 

415 handleDict : `dict` 

416 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

417 handle dictionary with keys: 

418 

419 ``"camera"`` 

420 Camera object (`lsst.afw.cameraGeom.Camera`) 

421 ``"fgcmLookUpTable"`` 

422 handle for the FGCM look-up table. 

423 tract : `int` 

424 Tract number 

425 visitCat : `lsst.afw.table.BaseCatalog` 

426 FGCM visitCat from `FgcmBuildStarsTask` 

427 zptCat : `lsst.afw.table.BaseCatalog` 

428 FGCM zeropoint catalog from `FgcmFitCycleTask` 

429 atmCat : `lsst.afw.table.BaseCatalog` 

430 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

431 stdCat : `lsst.afw.table.SimpleCatalog` 

432 FGCM standard star catalog from `FgcmFitCycleTask` 

433 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig` 

434 Configuration object from `FgcmBuildStarsTask` 

435 

436 Returns 

437 ------- 

438 retStruct : `lsst.pipe.base.Struct` 

439 Output structure with keys: 

440 

441 offsets : `np.ndarray` 

442 Final reference offsets, per band. 

443 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

444 Generator that returns (visit, transmissionCurve) tuples. 

445 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

446 Generator that returns (visit, exposureCatalog) tuples. 

447 """ 

448 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

449 

450 md = stdCat.getMetadata() 

451 bands = md.getArray('BANDS') 

452 

453 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

454 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

455 "in fgcmBuildStarsTask.") 

456 

457 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

458 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

459 

460 if self.config.doReferenceCalibration: 

461 lutCat = handleDict['fgcmLookUpTable'].get() 

462 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap) 

463 else: 

464 offsets = np.zeros(len(bands)) 

465 

466 if self.config.doZeropointOutput: 

467 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands, 

468 physicalFilterMap) 

469 else: 

470 pcgen = None 

471 

472 if self.config.doAtmosphereOutput: 

473 atmgen = self._outputAtmospheres(handleDict, atmCat) 

474 else: 

475 atmgen = None 

476 

477 retStruct = pipeBase.Struct(offsets=offsets, 

478 atmospheres=atmgen) 

479 retStruct.photoCalibCatalogs = pcgen 

480 

481 return retStruct 

482 

483 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands): 

484 """ 

485 Compute offsets relative to a reference catalog. 

486 

487 This method splits the star catalog into healpix pixels 

488 and computes the calibration transfer for a sample of 

489 these pixels to approximate the 'absolute' calibration 

490 values (on for each band) to apply to transfer the 

491 absolute scale. 

492 

493 Parameters 

494 ---------- 

495 stdCat : `lsst.afw.table.SimpleCatalog` 

496 FGCM standard stars 

497 lutCat : `lsst.afw.table.SimpleCatalog` 

498 FGCM Look-up table 

499 physicalFilterMap : `dict` 

500 Dictionary of mappings from physical filter to FGCM band. 

501 bands : `list` [`str`] 

502 List of band names from FGCM output 

503 Returns 

504 ------- 

505 offsets : `numpy.array` of floats 

506 Per band zeropoint offsets 

507 """ 

508 

509 # Only use stars that are observed in all the bands that were actually used 

510 # This will ensure that we use the same healpix pixels for the absolute 

511 # calibration of each band. 

512 minObs = stdCat['ngood'].min(axis=1) 

513 

514 goodStars = (minObs >= 1) 

515 stdCat = stdCat[goodStars] 

516 

517 self.log.info("Found %d stars with at least 1 good observation in each band" % 

518 (len(stdCat))) 

519 

520 # Associate each band with the appropriate physicalFilter and make 

521 # filterLabels 

522 filterLabels = [] 

523 

524 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',') 

525 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',') 

526 physicalFilterMapBands = list(physicalFilterMap.values()) 

527 physicalFilterMapFilters = list(physicalFilterMap.keys()) 

528 for band in bands: 

529 # Find a physical filter associated from the band by doing 

530 # a reverse lookup on the physicalFilterMap dict 

531 physicalFilterMapIndex = physicalFilterMapBands.index(band) 

532 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex] 

533 # Find the appropriate fgcm standard physicalFilter 

534 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter) 

535 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex] 

536 filterLabels.append(afwImage.FilterLabel(band=band, 

537 physical=stdPhysicalFilter)) 

538 

539 # We have to make a table for each pixel with flux/fluxErr 

540 # This is a temporary table generated for input to the photoCal task. 

541 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

542 # have had chromatic corrections applied to get to the standard system 

543 # specified by the atmosphere/instrumental parameters), nor are they 

544 # in Jansky (since they don't have a proper absolute calibration: the overall 

545 # zeropoint is estimated from the telescope size, etc.) 

546 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

547 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

548 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

549 doc="instrumental flux (counts)") 

550 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

551 doc="instrumental flux error (counts)") 

552 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

553 type='Flag', 

554 doc="bad flag") 

555 

556 # Split up the stars 

557 # Note that there is an assumption here that the ra/dec coords stored 

558 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

559 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

560 ipring = hpg.angle_to_pixel( 

561 self.config.referencePixelizationNside, 

562 stdCat['coord_ra'], 

563 stdCat['coord_dec'], 

564 degrees=False, 

565 ) 

566 h, rev = esutil.stat.histogram(ipring, rev=True) 

567 

568 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

569 

570 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

571 (gdpix.size, 

572 self.config.referencePixelizationNside, 

573 self.config.referencePixelizationMinStars)) 

574 

575 if gdpix.size < self.config.referencePixelizationNPixels: 

576 self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

577 (gdpix.size, self.config.referencePixelizationNPixels)) 

578 else: 

579 # Sample out the pixels we want to use 

580 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

581 

582 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

583 ('nstar', 'i4', len(bands)), 

584 ('nmatch', 'i4', len(bands)), 

585 ('zp', 'f4', len(bands)), 

586 ('zpErr', 'f4', len(bands))]) 

587 results['hpix'] = ipring[rev[rev[gdpix]]] 

588 

589 # We need a boolean index to deal with catalogs... 

590 selected = np.zeros(len(stdCat), dtype=bool) 

591 

592 refFluxFields = [None]*len(bands) 

593 

594 for p_index, pix in enumerate(gdpix): 

595 i1a = rev[rev[pix]: rev[pix + 1]] 

596 

597 # the stdCat afwTable can only be indexed with boolean arrays, 

598 # and not numpy index arrays (see DM-16497). This little trick 

599 # converts the index array into a boolean array 

600 selected[:] = False 

601 selected[i1a] = True 

602 

603 for b_index, filterLabel in enumerate(filterLabels): 

604 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index, 

605 filterLabel, stdCat, 

606 selected, refFluxFields) 

607 results['nstar'][p_index, b_index] = len(i1a) 

608 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag) 

609 results['zp'][p_index, b_index] = struct.zp 

610 results['zpErr'][p_index, b_index] = struct.sigma 

611 

612 # And compute the summary statistics 

613 offsets = np.zeros(len(bands)) 

614 

615 for b_index, band in enumerate(bands): 

616 # make configurable 

617 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch) 

618 offsets[b_index] = np.median(results['zp'][ok, b_index]) 

619 # use median absolute deviation to estimate Normal sigma 

620 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

621 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index])) 

622 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f", 

623 band, offsets[b_index], madSigma) 

624 

625 return offsets 

626 

627 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

628 b_index, filterLabel, stdCat, selected, refFluxFields): 

629 """ 

630 Compute the zeropoint offset between the fgcm stdCat and the reference 

631 stars for one pixel in one band 

632 

633 Parameters 

634 ---------- 

635 sourceMapper : `lsst.afw.table.SchemaMapper` 

636 Mapper to go from stdCat to calibratable catalog 

637 badStarKey : `lsst.afw.table.Key` 

638 Key for the field with bad stars 

639 b_index : `int` 

640 Index of the band in the star catalog 

641 filterLabel : `lsst.afw.image.FilterLabel` 

642 filterLabel with band and physical filter 

643 stdCat : `lsst.afw.table.SimpleCatalog` 

644 FGCM standard stars 

645 selected : `numpy.array(dtype=bool)` 

646 Boolean array of which stars are in the pixel 

647 refFluxFields : `list` 

648 List of names of flux fields for reference catalog 

649 """ 

650 

651 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

652 sourceCat.reserve(selected.sum()) 

653 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

654 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5)) 

655 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index] 

656 * sourceCat['instFlux']) 

657 # Make sure we only use stars that have valid measurements 

658 # (This is perhaps redundant with requirements above that the 

659 # stars be observed in all bands, but it can't hurt) 

660 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0) 

661 for rec in sourceCat[badStar]: 

662 rec.set(badStarKey, True) 

663 

664 exposure = afwImage.ExposureF() 

665 exposure.setFilter(filterLabel) 

666 

667 if refFluxFields[b_index] is None: 

668 # Need to find the flux field in the reference catalog 

669 # to work around limitations of DirectMatch in PhotoCal 

670 ctr = stdCat[0].getCoord() 

671 rad = 0.05*lsst.geom.degrees 

672 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel) 

673 refFluxFields[b_index] = refDataTest.fluxField 

674 

675 # Make a copy of the config so that we can modify it 

676 calConfig = copy.copy(self.config.photoCal.value) 

677 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index] 

678 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err' 

679 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

680 config=calConfig, 

681 schema=sourceCat.getSchema()) 

682 

683 struct = calTask.run(exposure, sourceCat) 

684 

685 return struct 

686 

687 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands, 

688 physicalFilterMap, tract=None): 

689 """Output the zeropoints in fgcm_photoCalib format. 

690 

691 Parameters 

692 ---------- 

693 camera : `lsst.afw.cameraGeom.Camera` 

694 Camera from the butler. 

695 zptCat : `lsst.afw.table.BaseCatalog` 

696 FGCM zeropoint catalog from `FgcmFitCycleTask`. 

697 visitCat : `lsst.afw.table.BaseCatalog` 

698 FGCM visitCat from `FgcmBuildStarsTask`. 

699 offsets : `numpy.array` 

700 Float array of absolute calibration offsets, one for each filter. 

701 bands : `list` [`str`] 

702 List of band names from FGCM output. 

703 physicalFilterMap : `dict` 

704 Dictionary of mappings from physical filter to FGCM band. 

705 tract: `int`, optional 

706 Tract number to output. Default is None (global calibration) 

707 

708 Returns 

709 ------- 

710 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

711 Generator that returns (visit, exposureCatalog) tuples. 

712 """ 

713 # Select visit/ccds where we have a calibration 

714 # This includes ccds where we were able to interpolate from neighboring 

715 # ccds. 

716 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

717 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) 

718 & (zptCat['fgcmZptVar'] > 0.0) 

719 & (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE)) 

720 

721 # Log warnings for any visit which has no valid zeropoints 

722 badVisits = np.unique(zptCat['visit'][~selected]) 

723 goodVisits = np.unique(zptCat['visit'][selected]) 

724 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

725 for allBadVisit in allBadVisits: 

726 self.log.warning(f'No suitable photoCalib for visit {allBadVisit}') 

727 

728 # Get a mapping from filtername to the offsets 

729 offsetMapping = {} 

730 for f in physicalFilterMap: 

731 # Not every filter in the map will necesarily have a band. 

732 if physicalFilterMap[f] in bands: 

733 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])] 

734 

735 # Get a mapping from "ccd" to the ccd index used for the scaling 

736 ccdMapping = {} 

737 for ccdIndex, detector in enumerate(camera): 

738 ccdMapping[detector.getId()] = ccdIndex 

739 

740 # And a mapping to get the flat-field scaling values 

741 scalingMapping = {} 

742 for rec in visitCat: 

743 scalingMapping[rec['visit']] = rec['scaling'] 

744 

745 if self.config.doComposeWcsJacobian: 

746 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

747 

748 # The zptCat is sorted by visit, which is useful 

749 lastVisit = -1 

750 zptVisitCatalog = None 

751 

752 metadata = dafBase.PropertyList() 

753 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

754 metadata.add("COMMENT", "Only detectors with data have entries.") 

755 

756 for rec in zptCat[selected]: 

757 # Retrieve overall scaling 

758 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]] 

759 

760 # The postCalibrationOffset describe any zeropoint offsets 

761 # to apply after the fgcm calibration. The first part comes 

762 # from the reference catalog match (used in testing). The 

763 # second part comes from the mean chromatic correction 

764 # (if configured). 

765 postCalibrationOffset = offsetMapping[rec['filtername']] 

766 if self.config.doApplyMeanChromaticCorrection: 

767 postCalibrationOffset += rec['fgcmDeltaChrom'] 

768 

769 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

770 rec['fgcmfZptChebXyMax']) 

771 # Convert from FGCM AB to nJy 

772 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

773 rec['fgcmfZptChebXyMax'], 

774 offset=postCalibrationOffset, 

775 scaling=scaling) 

776 

777 if self.config.doComposeWcsJacobian: 

778 

779 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']], 

780 fgcmSuperStarField, 

781 fgcmZptField]) 

782 else: 

783 # The photoCalib is just the product of the fgcmSuperStarField and the 

784 # fgcmZptField 

785 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

786 

787 # The "mean" calibration will be set to the center of the ccd for reference 

788 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

789 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

790 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

791 calibrationErr=calibErr, 

792 calibration=fgcmField, 

793 isConstant=False) 

794 

795 # Return full per-visit exposure catalogs 

796 if rec['visit'] != lastVisit: 

797 # This is a new visit. If the last visit was not -1, yield 

798 # the ExposureCatalog 

799 if lastVisit > -1: 

800 # ensure that the detectors are in sorted order, for fast lookups 

801 zptVisitCatalog.sort() 

802 yield (int(lastVisit), zptVisitCatalog) 

803 else: 

804 # We need to create a new schema 

805 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema() 

806 zptExpCatSchema.addField('visit', type='L', doc='Visit number') 

807 

808 # And start a new one 

809 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema) 

810 zptVisitCatalog.setMetadata(metadata) 

811 

812 lastVisit = int(rec['visit']) 

813 

814 catRecord = zptVisitCatalog.addNew() 

815 catRecord['id'] = int(rec['detector']) 

816 catRecord['visit'] = rec['visit'] 

817 catRecord.setPhotoCalib(photoCalib) 

818 

819 # Final output of last exposure catalog 

820 # ensure that the detectors are in sorted order, for fast lookups 

821 zptVisitCatalog.sort() 

822 yield (int(lastVisit), zptVisitCatalog) 

823 

824 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

825 """ 

826 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

827 and scaling. 

828 

829 Parameters 

830 ---------- 

831 coefficients: `numpy.array` 

832 Flattened array of chebyshev coefficients 

833 xyMax: `list` of length 2 

834 Maximum x and y of the chebyshev bounding box 

835 offset: `float`, optional 

836 Absolute calibration offset. Default is 0.0 

837 scaling: `float`, optional 

838 Flat scaling value from fgcmBuildStars. Default is 1.0 

839 

840 Returns 

841 ------- 

842 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

843 """ 

844 

845 orderPlus1 = int(np.sqrt(coefficients.size)) 

846 pars = np.zeros((orderPlus1, orderPlus1)) 

847 

848 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

849 lsst.geom.Point2I(*xyMax)) 

850 

851 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) 

852 * (10.**(offset/-2.5))*scaling) 

853 

854 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

855 

856 return boundedField 

857 

858 def _outputAtmospheres(self, handleDict, atmCat): 

859 """ 

860 Output the atmospheres. 

861 

862 Parameters 

863 ---------- 

864 handleDict : `dict` 

865 All data handles are `lsst.daf.butler.DeferredDatasetHandle` 

866 The handleDict has the follownig keys: 

867 

868 ``"fgcmLookUpTable"`` 

869 handle for the FGCM look-up table. 

870 atmCat : `lsst.afw.table.BaseCatalog` 

871 FGCM atmosphere parameter catalog from fgcmFitCycleTask. 

872 

873 Returns 

874 ------- 

875 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

876 Generator that returns (visit, transmissionCurve) tuples. 

877 """ 

878 # First, we need to grab the look-up table and key info 

879 lutCat = handleDict['fgcmLookUpTable'].get() 

880 

881 atmosphereTableName = lutCat[0]['tablename'] 

882 elevation = lutCat[0]['elevation'] 

883 atmLambda = lutCat[0]['atmLambda'] 

884 lutCat = None 

885 

886 # Make the atmosphere table if possible 

887 try: 

888 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

889 atmTable.loadTable() 

890 except IOError: 

891 atmTable = None 

892 

893 if atmTable is None: 

894 # Try to use MODTRAN instead 

895 try: 

896 modGen = fgcm.ModtranGenerator(elevation) 

897 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

898 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

899 except (ValueError, IOError) as e: 

900 raise RuntimeError("FGCM look-up-table generated with modtran, " 

901 "but modtran not configured to run.") from e 

902 

903 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

904 

905 for i, visit in enumerate(atmCat['visit']): 

906 if atmTable is not None: 

907 # Interpolate the atmosphere table 

908 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

909 pwv=atmCat[i]['pwv'], 

910 o3=atmCat[i]['o3'], 

911 tau=atmCat[i]['tau'], 

912 alpha=atmCat[i]['alpha'], 

913 zenith=zenith[i], 

914 ctranslamstd=[atmCat[i]['cTrans'], 

915 atmCat[i]['lamStd']]) 

916 else: 

917 # Run modtran 

918 modAtm = modGen(pmb=atmCat[i]['pmb'], 

919 pwv=atmCat[i]['pwv'], 

920 o3=atmCat[i]['o3'], 

921 tau=atmCat[i]['tau'], 

922 alpha=atmCat[i]['alpha'], 

923 zenith=zenith[i], 

924 lambdaRange=lambdaRange, 

925 lambdaStep=lambdaStep, 

926 ctranslamstd=[atmCat[i]['cTrans'], 

927 atmCat[i]['lamStd']]) 

928 atmVals = modAtm['COMBINED'] 

929 

930 # Now need to create something to persist... 

931 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

932 wavelengths=atmLambda, 

933 throughputAtMin=atmVals[0], 

934 throughputAtMax=atmVals[-1]) 

935 

936 yield (int(visit), curve)