Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py: 17%

343 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-01 02:26 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34import copy 

35 

36import numpy as np 

37import hpgeom as hpg 

38import esutil 

39from astropy import units 

40 

41import lsst.daf.base as dafBase 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45from lsst.afw.image import TransmissionCurve 

46from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

47from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

48from lsst.pipe.tasks.photoCal import PhotoCalTask 

49import lsst.geom 

50import lsst.afw.image as afwImage 

51import lsst.afw.math as afwMath 

52import lsst.afw.table as afwTable 

53from lsst.meas.algorithms import DatasetConfig 

54from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

55 

56from .utilities import computeApproxPixelAreaFields 

57from .utilities import lookupStaticCalibrations 

58from .utilities import FGCM_ILLEGAL_VALUE 

59 

60import fgcm 

61 

62__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask'] 

63 

64 

65class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections, 

66 dimensions=("instrument",), 

67 defaultTemplates={"cycleNumber": "0"}): 

68 camera = connectionTypes.PrerequisiteInput( 

69 doc="Camera instrument", 

70 name="camera", 

71 storageClass="Camera", 

72 dimensions=("instrument",), 

73 lookupFunction=lookupStaticCalibrations, 

74 isCalibration=True, 

75 ) 

76 

77 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

78 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

79 "chromatic corrections."), 

80 name="fgcmLookUpTable", 

81 storageClass="Catalog", 

82 dimensions=("instrument",), 

83 deferLoad=True, 

84 ) 

85 

86 fgcmVisitCatalog = connectionTypes.Input( 

87 doc="Catalog of visit information for fgcm", 

88 name="fgcmVisitCatalog", 

89 storageClass="Catalog", 

90 dimensions=("instrument",), 

91 deferLoad=True, 

92 ) 

93 

94 fgcmStandardStars = connectionTypes.Input( 

95 doc="Catalog of standard star data from fgcm fit", 

96 name="fgcmStandardStars{cycleNumber}", 

97 storageClass="SimpleCatalog", 

98 dimensions=("instrument",), 

99 deferLoad=True, 

100 ) 

101 

102 fgcmZeropoints = connectionTypes.Input( 

103 doc="Catalog of zeropoints from fgcm fit", 

104 name="fgcmZeropoints{cycleNumber}", 

105 storageClass="Catalog", 

106 dimensions=("instrument",), 

107 deferLoad=True, 

108 ) 

109 

110 fgcmAtmosphereParameters = connectionTypes.Input( 

111 doc="Catalog of atmosphere parameters from fgcm fit", 

112 name="fgcmAtmosphereParameters{cycleNumber}", 

113 storageClass="Catalog", 

114 dimensions=("instrument",), 

115 deferLoad=True, 

116 ) 

117 

118 refCat = connectionTypes.PrerequisiteInput( 

119 doc="Reference catalog to use for photometric calibration", 

120 name="cal_ref_cat", 

121 storageClass="SimpleCatalog", 

122 dimensions=("skypix",), 

123 deferLoad=True, 

124 multiple=True, 

125 ) 

126 

127 fgcmPhotoCalib = connectionTypes.Output( 

128 doc=("Per-visit photometric calibrations derived from fgcm calibration. " 

129 "These catalogs use detector id for the id and are sorted for " 

130 "fast lookups of a detector."), 

131 name="fgcmPhotoCalibCatalog", 

132 storageClass="ExposureCatalog", 

133 dimensions=("instrument", "visit",), 

134 multiple=True, 

135 ) 

136 

137 fgcmTransmissionAtmosphere = connectionTypes.Output( 

138 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

139 name="transmission_atmosphere_fgcm", 

140 storageClass="TransmissionCurve", 

141 dimensions=("instrument", 

142 "visit",), 

143 multiple=True, 

144 ) 

145 

146 fgcmOffsets = connectionTypes.Output( 

147 doc="Per-band offsets computed from doReferenceCalibration", 

148 name="fgcmReferenceCalibrationOffsets", 

149 storageClass="Catalog", 

150 dimensions=("instrument",), 

151 multiple=False, 

152 ) 

153 

154 def __init__(self, *, config=None): 

155 super().__init__(config=config) 

156 

157 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

158 raise ValueError("cycleNumber must be of integer format") 

159 

160 if not config.doReferenceCalibration: 

161 self.prerequisiteInputs.remove("refCat") 

162 if not config.doAtmosphereOutput: 

163 self.inputs.remove("fgcmAtmosphereParameters") 

164 if not config.doZeropointOutput: 

165 self.inputs.remove("fgcmZeropoints") 

166 if not config.doReferenceCalibration: 

167 self.outputs.remove("fgcmOffsets") 

168 

169 

170class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig, 

171 pipelineConnections=FgcmOutputProductsConnections): 

172 """Config for FgcmOutputProductsTask""" 

173 

174 cycleNumber = pexConfig.Field( 

175 doc="Final fit cycle from FGCM fit", 

176 dtype=int, 

177 default=None, 

178 ) 

179 physicalFilterMap = pexConfig.DictField( 

180 doc="Mapping from 'physicalFilter' to band.", 

181 keytype=str, 

182 itemtype=str, 

183 default={}, 

184 ) 

185 # The following fields refer to calibrating from a reference 

186 # catalog, but in the future this might need to be expanded 

187 doReferenceCalibration = pexConfig.Field( 

188 doc=("Transfer 'absolute' calibration from reference catalog? " 

189 "This afterburner step is unnecessary if reference stars " 

190 "were used in the full fit in FgcmFitCycleTask."), 

191 dtype=bool, 

192 default=False, 

193 ) 

194 doRefcatOutput = pexConfig.Field( 

195 doc="Output standard stars in reference catalog format", 

196 dtype=bool, 

197 default=False, 

198 deprecated="doRefcatOutput is no longer supported; this config will be removed after v24" 

199 ) 

200 doAtmosphereOutput = pexConfig.Field( 

201 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

202 dtype=bool, 

203 default=True, 

204 ) 

205 doZeropointOutput = pexConfig.Field( 

206 doc="Output zeropoints in fgcm_photoCalib format", 

207 dtype=bool, 

208 default=True, 

209 ) 

210 doComposeWcsJacobian = pexConfig.Field( 

211 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

212 dtype=bool, 

213 default=True, 

214 ) 

215 doApplyMeanChromaticCorrection = pexConfig.Field( 

216 doc="Apply the mean chromatic correction to the zeropoints?", 

217 dtype=bool, 

218 default=True, 

219 ) 

220 refObjLoader = pexConfig.ConfigurableField( 

221 target=LoadIndexedReferenceObjectsTask, 

222 doc="reference object loader for 'absolute' photometric calibration", 

223 deprecated="refObjLoader is deprecated, and will be removed after v24", 

224 ) 

225 photoCal = pexConfig.ConfigurableField( 

226 target=PhotoCalTask, 

227 doc="task to perform 'absolute' calibration", 

228 ) 

229 referencePixelizationNside = pexConfig.Field( 

230 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

231 dtype=int, 

232 default=64, 

233 ) 

234 referencePixelizationMinStars = pexConfig.Field( 

235 doc=("Minimum number of stars per healpix pixel to select for comparison" 

236 "to the specified reference catalog"), 

237 dtype=int, 

238 default=200, 

239 ) 

240 referenceMinMatch = pexConfig.Field( 

241 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

242 dtype=int, 

243 default=50, 

244 ) 

245 referencePixelizationNPixels = pexConfig.Field( 

246 doc=("Number of healpix pixels to sample to do comparison. " 

247 "Doing too many will take a long time and not yield any more " 

248 "precise results because the final number is the median offset " 

249 "(per band) from the set of pixels."), 

250 dtype=int, 

251 default=100, 

252 ) 

253 datasetConfig = pexConfig.ConfigField( 

254 dtype=DatasetConfig, 

255 doc="Configuration for writing/reading ingested catalog", 

256 deprecated="The datasetConfig was only used for gen2; this config will be removed after v24.", 

257 ) 

258 

259 def setDefaults(self): 

260 pexConfig.Config.setDefaults(self) 

261 

262 # In order to transfer the "absolute" calibration from a reference 

263 # catalog to the relatively calibrated FGCM standard stars (one number 

264 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

265 # pixels. These basic settings ensure that only well-measured, good stars 

266 # from the source and reference catalogs are used for the matching. 

267 

268 # applyColorTerms needs to be False if doReferenceCalibration is False, 

269 # as is the new default after DM-16702 

270 self.photoCal.applyColorTerms = False 

271 self.photoCal.fluxField = 'instFlux' 

272 self.photoCal.magErrFloor = 0.003 

273 self.photoCal.match.referenceSelection.doSignalToNoise = True 

274 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

275 self.photoCal.match.sourceSelection.doSignalToNoise = True 

276 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

277 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

278 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

279 self.photoCal.match.sourceSelection.doFlags = True 

280 self.photoCal.match.sourceSelection.flags.good = [] 

281 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

282 self.photoCal.match.sourceSelection.doUnresolved = False 

283 

284 def validate(self): 

285 super().validate() 

286 

287 # Force the connections to conform with cycleNumber 

288 self.connections.cycleNumber = str(self.cycleNumber) 

289 

290 

291class FgcmOutputProductsTask(pipeBase.PipelineTask): 

292 """ 

293 Output products from FGCM global calibration. 

294 """ 

295 

296 ConfigClass = FgcmOutputProductsConfig 

297 _DefaultName = "fgcmOutputProducts" 

298 

299 def __init__(self, **kwargs): 

300 super().__init__(**kwargs) 

301 

302 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

303 handleDict = {} 

304 handleDict['camera'] = butlerQC.get(inputRefs.camera) 

305 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

306 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

307 handleDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars) 

308 

309 if self.config.doZeropointOutput: 

310 handleDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints) 

311 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

312 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

313 

314 if self.config.doAtmosphereOutput: 

315 handleDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters) 

316 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

317 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

318 

319 if self.config.doReferenceCalibration: 

320 refConfig = LoadReferenceObjectsConfig() 

321 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

322 for ref in inputRefs.refCat], 

323 refCats=butlerQC.get(inputRefs.refCat), 

324 log=self.log, 

325 config=refConfig) 

326 else: 

327 self.refObjLoader = None 

328 

329 struct = self.run(handleDict, self.config.physicalFilterMap) 

330 

331 # Output the photoCalib exposure catalogs 

332 if struct.photoCalibCatalogs is not None: 

333 self.log.info("Outputting photoCalib catalogs.") 

334 for visit, expCatalog in struct.photoCalibCatalogs: 

335 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

336 self.log.info("Done outputting photoCalib catalogs.") 

337 

338 # Output the atmospheres 

339 if struct.atmospheres is not None: 

340 self.log.info("Outputting atmosphere transmission files.") 

341 for visit, atm in struct.atmospheres: 

342 butlerQC.put(atm, atmRefDict[visit]) 

343 self.log.info("Done outputting atmosphere files.") 

344 

345 if self.config.doReferenceCalibration: 

346 # Turn offset into simple catalog for persistence if necessary 

347 schema = afwTable.Schema() 

348 schema.addField('offset', type=np.float64, 

349 doc="Post-process calibration offset (mag)") 

350 offsetCat = afwTable.BaseCatalog(schema) 

351 offsetCat.resize(len(struct.offsets)) 

352 offsetCat['offset'][:] = struct.offsets 

353 

354 butlerQC.put(offsetCat, outputRefs.fgcmOffsets) 

355 

356 return 

357 

358 def run(self, handleDict, physicalFilterMap): 

359 """Run the output products task. 

360 

361 Parameters 

362 ---------- 

363 handleDict : `dict` 

364 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

365 handle dictionary with keys: 

366 

367 ``"camera"`` 

368 Camera object (`lsst.afw.cameraGeom.Camera`) 

369 ``"fgcmLookUpTable"`` 

370 handle for the FGCM look-up table. 

371 ``"fgcmVisitCatalog"`` 

372 handle for visit summary catalog. 

373 ``"fgcmStandardStars"`` 

374 handle for the output standard star catalog. 

375 ``"fgcmZeropoints"`` 

376 handle for the zeropoint data catalog. 

377 ``"fgcmAtmosphereParameters"`` 

378 handle for the atmosphere parameter catalog. 

379 ``"fgcmBuildStarsTableConfig"`` 

380 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`. 

381 physicalFilterMap : `dict` 

382 Dictionary of mappings from physical filter to FGCM band. 

383 

384 Returns 

385 ------- 

386 retStruct : `lsst.pipe.base.Struct` 

387 Output structure with keys: 

388 

389 offsets : `np.ndarray` 

390 Final reference offsets, per band. 

391 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

392 Generator that returns (visit, transmissionCurve) tuples. 

393 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

394 Generator that returns (visit, exposureCatalog) tuples. 

395 """ 

396 stdCat = handleDict['fgcmStandardStars'].get() 

397 md = stdCat.getMetadata() 

398 bands = md.getArray('BANDS') 

399 

400 if self.config.doReferenceCalibration: 

401 lutCat = handleDict['fgcmLookUpTable'].get() 

402 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands) 

403 else: 

404 offsets = np.zeros(len(bands)) 

405 

406 del stdCat 

407 

408 if self.config.doZeropointOutput: 

409 zptCat = handleDict['fgcmZeropoints'].get() 

410 visitCat = handleDict['fgcmVisitCatalog'].get() 

411 

412 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands, 

413 physicalFilterMap) 

414 else: 

415 pcgen = None 

416 

417 if self.config.doAtmosphereOutput: 

418 atmCat = handleDict['fgcmAtmosphereParameters'].get() 

419 atmgen = self._outputAtmospheres(handleDict, atmCat) 

420 else: 

421 atmgen = None 

422 

423 retStruct = pipeBase.Struct(offsets=offsets, 

424 atmospheres=atmgen) 

425 retStruct.photoCalibCatalogs = pcgen 

426 

427 return retStruct 

428 

429 def generateTractOutputProducts(self, handleDict, tract, 

430 visitCat, zptCat, atmCat, stdCat, 

431 fgcmBuildStarsConfig): 

432 """ 

433 Generate the output products for a given tract, as specified in the config. 

434 

435 This method is here to have an alternate entry-point for 

436 FgcmCalibrateTract. 

437 

438 Parameters 

439 ---------- 

440 handleDict : `dict` 

441 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

442 handle dictionary with keys: 

443 

444 ``"camera"`` 

445 Camera object (`lsst.afw.cameraGeom.Camera`) 

446 ``"fgcmLookUpTable"`` 

447 handle for the FGCM look-up table. 

448 tract : `int` 

449 Tract number 

450 visitCat : `lsst.afw.table.BaseCatalog` 

451 FGCM visitCat from `FgcmBuildStarsTask` 

452 zptCat : `lsst.afw.table.BaseCatalog` 

453 FGCM zeropoint catalog from `FgcmFitCycleTask` 

454 atmCat : `lsst.afw.table.BaseCatalog` 

455 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

456 stdCat : `lsst.afw.table.SimpleCatalog` 

457 FGCM standard star catalog from `FgcmFitCycleTask` 

458 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig` 

459 Configuration object from `FgcmBuildStarsTask` 

460 

461 Returns 

462 ------- 

463 retStruct : `lsst.pipe.base.Struct` 

464 Output structure with keys: 

465 

466 offsets : `np.ndarray` 

467 Final reference offsets, per band. 

468 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

469 Generator that returns (visit, transmissionCurve) tuples. 

470 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

471 Generator that returns (visit, exposureCatalog) tuples. 

472 """ 

473 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

474 

475 md = stdCat.getMetadata() 

476 bands = md.getArray('BANDS') 

477 

478 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

479 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

480 "in fgcmBuildStarsTask.") 

481 

482 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

483 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

484 

485 if self.config.doReferenceCalibration: 

486 lutCat = handleDict['fgcmLookUpTable'].get() 

487 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap) 

488 else: 

489 offsets = np.zeros(len(bands)) 

490 

491 if self.config.doZeropointOutput: 

492 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands, 

493 physicalFilterMap) 

494 else: 

495 pcgen = None 

496 

497 if self.config.doAtmosphereOutput: 

498 atmgen = self._outputAtmospheres(handleDict, atmCat) 

499 else: 

500 atmgen = None 

501 

502 retStruct = pipeBase.Struct(offsets=offsets, 

503 atmospheres=atmgen) 

504 retStruct.photoCalibCatalogs = pcgen 

505 

506 return retStruct 

507 

508 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands): 

509 """ 

510 Compute offsets relative to a reference catalog. 

511 

512 This method splits the star catalog into healpix pixels 

513 and computes the calibration transfer for a sample of 

514 these pixels to approximate the 'absolute' calibration 

515 values (on for each band) to apply to transfer the 

516 absolute scale. 

517 

518 Parameters 

519 ---------- 

520 stdCat : `lsst.afw.table.SimpleCatalog` 

521 FGCM standard stars 

522 lutCat : `lsst.afw.table.SimpleCatalog` 

523 FGCM Look-up table 

524 physicalFilterMap : `dict` 

525 Dictionary of mappings from physical filter to FGCM band. 

526 bands : `list` [`str`] 

527 List of band names from FGCM output 

528 Returns 

529 ------- 

530 offsets : `numpy.array` of floats 

531 Per band zeropoint offsets 

532 """ 

533 

534 # Only use stars that are observed in all the bands that were actually used 

535 # This will ensure that we use the same healpix pixels for the absolute 

536 # calibration of each band. 

537 minObs = stdCat['ngood'].min(axis=1) 

538 

539 goodStars = (minObs >= 1) 

540 stdCat = stdCat[goodStars] 

541 

542 self.log.info("Found %d stars with at least 1 good observation in each band" % 

543 (len(stdCat))) 

544 

545 # Associate each band with the appropriate physicalFilter and make 

546 # filterLabels 

547 filterLabels = [] 

548 

549 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',') 

550 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',') 

551 physicalFilterMapBands = list(physicalFilterMap.values()) 

552 physicalFilterMapFilters = list(physicalFilterMap.keys()) 

553 for band in bands: 

554 # Find a physical filter associated from the band by doing 

555 # a reverse lookup on the physicalFilterMap dict 

556 physicalFilterMapIndex = physicalFilterMapBands.index(band) 

557 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex] 

558 # Find the appropriate fgcm standard physicalFilter 

559 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter) 

560 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex] 

561 filterLabels.append(afwImage.FilterLabel(band=band, 

562 physical=stdPhysicalFilter)) 

563 

564 # We have to make a table for each pixel with flux/fluxErr 

565 # This is a temporary table generated for input to the photoCal task. 

566 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

567 # have had chromatic corrections applied to get to the standard system 

568 # specified by the atmosphere/instrumental parameters), nor are they 

569 # in Jansky (since they don't have a proper absolute calibration: the overall 

570 # zeropoint is estimated from the telescope size, etc.) 

571 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

572 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

573 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

574 doc="instrumental flux (counts)") 

575 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

576 doc="instrumental flux error (counts)") 

577 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

578 type='Flag', 

579 doc="bad flag") 

580 

581 # Split up the stars 

582 # Note that there is an assumption here that the ra/dec coords stored 

583 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

584 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

585 ipring = hpg.angle_to_pixel( 

586 self.config.referencePixelizationNside, 

587 stdCat['coord_ra'], 

588 stdCat['coord_dec'], 

589 degrees=False, 

590 ) 

591 h, rev = esutil.stat.histogram(ipring, rev=True) 

592 

593 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

594 

595 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

596 (gdpix.size, 

597 self.config.referencePixelizationNside, 

598 self.config.referencePixelizationMinStars)) 

599 

600 if gdpix.size < self.config.referencePixelizationNPixels: 

601 self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

602 (gdpix.size, self.config.referencePixelizationNPixels)) 

603 else: 

604 # Sample out the pixels we want to use 

605 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

606 

607 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

608 ('nstar', 'i4', len(bands)), 

609 ('nmatch', 'i4', len(bands)), 

610 ('zp', 'f4', len(bands)), 

611 ('zpErr', 'f4', len(bands))]) 

612 results['hpix'] = ipring[rev[rev[gdpix]]] 

613 

614 # We need a boolean index to deal with catalogs... 

615 selected = np.zeros(len(stdCat), dtype=bool) 

616 

617 refFluxFields = [None]*len(bands) 

618 

619 for p_index, pix in enumerate(gdpix): 

620 i1a = rev[rev[pix]: rev[pix + 1]] 

621 

622 # the stdCat afwTable can only be indexed with boolean arrays, 

623 # and not numpy index arrays (see DM-16497). This little trick 

624 # converts the index array into a boolean array 

625 selected[:] = False 

626 selected[i1a] = True 

627 

628 for b_index, filterLabel in enumerate(filterLabels): 

629 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index, 

630 filterLabel, stdCat, 

631 selected, refFluxFields) 

632 results['nstar'][p_index, b_index] = len(i1a) 

633 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag) 

634 results['zp'][p_index, b_index] = struct.zp 

635 results['zpErr'][p_index, b_index] = struct.sigma 

636 

637 # And compute the summary statistics 

638 offsets = np.zeros(len(bands)) 

639 

640 for b_index, band in enumerate(bands): 

641 # make configurable 

642 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch) 

643 offsets[b_index] = np.median(results['zp'][ok, b_index]) 

644 # use median absolute deviation to estimate Normal sigma 

645 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

646 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index])) 

647 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f", 

648 band, offsets[b_index], madSigma) 

649 

650 return offsets 

651 

652 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

653 b_index, filterLabel, stdCat, selected, refFluxFields): 

654 """ 

655 Compute the zeropoint offset between the fgcm stdCat and the reference 

656 stars for one pixel in one band 

657 

658 Parameters 

659 ---------- 

660 sourceMapper : `lsst.afw.table.SchemaMapper` 

661 Mapper to go from stdCat to calibratable catalog 

662 badStarKey : `lsst.afw.table.Key` 

663 Key for the field with bad stars 

664 b_index : `int` 

665 Index of the band in the star catalog 

666 filterLabel : `lsst.afw.image.FilterLabel` 

667 filterLabel with band and physical filter 

668 stdCat : `lsst.afw.table.SimpleCatalog` 

669 FGCM standard stars 

670 selected : `numpy.array(dtype=bool)` 

671 Boolean array of which stars are in the pixel 

672 refFluxFields : `list` 

673 List of names of flux fields for reference catalog 

674 """ 

675 

676 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

677 sourceCat.reserve(selected.sum()) 

678 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

679 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5)) 

680 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index] 

681 * sourceCat['instFlux']) 

682 # Make sure we only use stars that have valid measurements 

683 # (This is perhaps redundant with requirements above that the 

684 # stars be observed in all bands, but it can't hurt) 

685 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0) 

686 for rec in sourceCat[badStar]: 

687 rec.set(badStarKey, True) 

688 

689 exposure = afwImage.ExposureF() 

690 exposure.setFilter(filterLabel) 

691 

692 if refFluxFields[b_index] is None: 

693 # Need to find the flux field in the reference catalog 

694 # to work around limitations of DirectMatch in PhotoCal 

695 ctr = stdCat[0].getCoord() 

696 rad = 0.05*lsst.geom.degrees 

697 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel) 

698 refFluxFields[b_index] = refDataTest.fluxField 

699 

700 # Make a copy of the config so that we can modify it 

701 calConfig = copy.copy(self.config.photoCal.value) 

702 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index] 

703 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err' 

704 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

705 config=calConfig, 

706 schema=sourceCat.getSchema()) 

707 

708 struct = calTask.run(exposure, sourceCat) 

709 

710 return struct 

711 

712 def _formatCatalog(self, fgcmStarCat, offsets, bands): 

713 """ 

714 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

715 

716 Parameters 

717 ---------- 

718 fgcmStarCat : `lsst.afw.Table.SimpleCatalog` 

719 SimpleCatalog as output by fgcmcal 

720 offsets : `list` with len(self.bands) entries 

721 Zeropoint offsets to apply 

722 bands : `list` [`str`] 

723 List of band names from FGCM output 

724 

725 Returns 

726 ------- 

727 formattedCat: `lsst.afw.table.SimpleCatalog` 

728 SimpleCatalog suitable for using as a reference catalog 

729 """ 

730 

731 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

732 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands, 

733 addCentroid=False, 

734 addIsResolved=True, 

735 coordErrDim=0) 

736 sourceMapper.addMinimalSchema(minSchema) 

737 for band in bands: 

738 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

739 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

740 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

741 

742 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

743 formattedCat.reserve(len(fgcmStarCat)) 

744 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

745 

746 # Note that we don't have to set `resolved` because the default is False 

747 

748 for b, band in enumerate(bands): 

749 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

750 # We want fluxes in nJy from calibrated AB magnitudes 

751 # (after applying offset). Updated after RFC-549 and RFC-575. 

752 flux = (mag*units.ABmag).to_value(units.nJy) 

753 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

754 

755 formattedCat['%s_flux' % (band)][:] = flux 

756 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

757 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

758 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

759 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

760 

761 addRefCatMetadata(formattedCat) 

762 

763 return formattedCat 

764 

765 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands, 

766 physicalFilterMap, tract=None): 

767 """Output the zeropoints in fgcm_photoCalib format. 

768 

769 Parameters 

770 ---------- 

771 camera : `lsst.afw.cameraGeom.Camera` 

772 Camera from the butler. 

773 zptCat : `lsst.afw.table.BaseCatalog` 

774 FGCM zeropoint catalog from `FgcmFitCycleTask`. 

775 visitCat : `lsst.afw.table.BaseCatalog` 

776 FGCM visitCat from `FgcmBuildStarsTask`. 

777 offsets : `numpy.array` 

778 Float array of absolute calibration offsets, one for each filter. 

779 bands : `list` [`str`] 

780 List of band names from FGCM output. 

781 physicalFilterMap : `dict` 

782 Dictionary of mappings from physical filter to FGCM band. 

783 tract: `int`, optional 

784 Tract number to output. Default is None (global calibration) 

785 

786 Returns 

787 ------- 

788 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

789 Generator that returns (visit, exposureCatalog) tuples. 

790 """ 

791 # Select visit/ccds where we have a calibration 

792 # This includes ccds where we were able to interpolate from neighboring 

793 # ccds. 

794 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

795 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) 

796 & (zptCat['fgcmZptVar'] > 0.0) 

797 & (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE)) 

798 

799 # Log warnings for any visit which has no valid zeropoints 

800 badVisits = np.unique(zptCat['visit'][~selected]) 

801 goodVisits = np.unique(zptCat['visit'][selected]) 

802 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

803 for allBadVisit in allBadVisits: 

804 self.log.warning(f'No suitable photoCalib for visit {allBadVisit}') 

805 

806 # Get a mapping from filtername to the offsets 

807 offsetMapping = {} 

808 for f in physicalFilterMap: 

809 # Not every filter in the map will necesarily have a band. 

810 if physicalFilterMap[f] in bands: 

811 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])] 

812 

813 # Get a mapping from "ccd" to the ccd index used for the scaling 

814 ccdMapping = {} 

815 for ccdIndex, detector in enumerate(camera): 

816 ccdMapping[detector.getId()] = ccdIndex 

817 

818 # And a mapping to get the flat-field scaling values 

819 scalingMapping = {} 

820 for rec in visitCat: 

821 scalingMapping[rec['visit']] = rec['scaling'] 

822 

823 if self.config.doComposeWcsJacobian: 

824 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

825 

826 # The zptCat is sorted by visit, which is useful 

827 lastVisit = -1 

828 zptVisitCatalog = None 

829 

830 metadata = dafBase.PropertyList() 

831 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

832 metadata.add("COMMENT", "Only detectors with data have entries.") 

833 

834 for rec in zptCat[selected]: 

835 # Retrieve overall scaling 

836 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]] 

837 

838 # The postCalibrationOffset describe any zeropoint offsets 

839 # to apply after the fgcm calibration. The first part comes 

840 # from the reference catalog match (used in testing). The 

841 # second part comes from the mean chromatic correction 

842 # (if configured). 

843 postCalibrationOffset = offsetMapping[rec['filtername']] 

844 if self.config.doApplyMeanChromaticCorrection: 

845 postCalibrationOffset += rec['fgcmDeltaChrom'] 

846 

847 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

848 rec['fgcmfZptChebXyMax']) 

849 # Convert from FGCM AB to nJy 

850 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

851 rec['fgcmfZptChebXyMax'], 

852 offset=postCalibrationOffset, 

853 scaling=scaling) 

854 

855 if self.config.doComposeWcsJacobian: 

856 

857 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']], 

858 fgcmSuperStarField, 

859 fgcmZptField]) 

860 else: 

861 # The photoCalib is just the product of the fgcmSuperStarField and the 

862 # fgcmZptField 

863 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

864 

865 # The "mean" calibration will be set to the center of the ccd for reference 

866 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

867 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

868 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

869 calibrationErr=calibErr, 

870 calibration=fgcmField, 

871 isConstant=False) 

872 

873 # Return full per-visit exposure catalogs 

874 if rec['visit'] != lastVisit: 

875 # This is a new visit. If the last visit was not -1, yield 

876 # the ExposureCatalog 

877 if lastVisit > -1: 

878 # ensure that the detectors are in sorted order, for fast lookups 

879 zptVisitCatalog.sort() 

880 yield (int(lastVisit), zptVisitCatalog) 

881 else: 

882 # We need to create a new schema 

883 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema() 

884 zptExpCatSchema.addField('visit', type='L', doc='Visit number') 

885 

886 # And start a new one 

887 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema) 

888 zptVisitCatalog.setMetadata(metadata) 

889 

890 lastVisit = int(rec['visit']) 

891 

892 catRecord = zptVisitCatalog.addNew() 

893 catRecord['id'] = int(rec['detector']) 

894 catRecord['visit'] = rec['visit'] 

895 catRecord.setPhotoCalib(photoCalib) 

896 

897 # Final output of last exposure catalog 

898 # ensure that the detectors are in sorted order, for fast lookups 

899 zptVisitCatalog.sort() 

900 yield (int(lastVisit), zptVisitCatalog) 

901 

902 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

903 """ 

904 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

905 and scaling. 

906 

907 Parameters 

908 ---------- 

909 coefficients: `numpy.array` 

910 Flattened array of chebyshev coefficients 

911 xyMax: `list` of length 2 

912 Maximum x and y of the chebyshev bounding box 

913 offset: `float`, optional 

914 Absolute calibration offset. Default is 0.0 

915 scaling: `float`, optional 

916 Flat scaling value from fgcmBuildStars. Default is 1.0 

917 

918 Returns 

919 ------- 

920 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

921 """ 

922 

923 orderPlus1 = int(np.sqrt(coefficients.size)) 

924 pars = np.zeros((orderPlus1, orderPlus1)) 

925 

926 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

927 lsst.geom.Point2I(*xyMax)) 

928 

929 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) 

930 * (10.**(offset/-2.5))*scaling) 

931 

932 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

933 

934 return boundedField 

935 

936 def _outputAtmospheres(self, handleDict, atmCat): 

937 """ 

938 Output the atmospheres. 

939 

940 Parameters 

941 ---------- 

942 handleDict : `dict` 

943 All data handles are `lsst.daf.butler.DeferredDatasetHandle` 

944 The handleDict has the follownig keys: 

945 

946 ``"fgcmLookUpTable"`` 

947 handle for the FGCM look-up table. 

948 atmCat : `lsst.afw.table.BaseCatalog` 

949 FGCM atmosphere parameter catalog from fgcmFitCycleTask. 

950 

951 Returns 

952 ------- 

953 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

954 Generator that returns (visit, transmissionCurve) tuples. 

955 """ 

956 # First, we need to grab the look-up table and key info 

957 lutCat = handleDict['fgcmLookUpTable'].get() 

958 

959 atmosphereTableName = lutCat[0]['tablename'] 

960 elevation = lutCat[0]['elevation'] 

961 atmLambda = lutCat[0]['atmLambda'] 

962 lutCat = None 

963 

964 # Make the atmosphere table if possible 

965 try: 

966 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

967 atmTable.loadTable() 

968 except IOError: 

969 atmTable = None 

970 

971 if atmTable is None: 

972 # Try to use MODTRAN instead 

973 try: 

974 modGen = fgcm.ModtranGenerator(elevation) 

975 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

976 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

977 except (ValueError, IOError) as e: 

978 raise RuntimeError("FGCM look-up-table generated with modtran, " 

979 "but modtran not configured to run.") from e 

980 

981 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

982 

983 for i, visit in enumerate(atmCat['visit']): 

984 if atmTable is not None: 

985 # Interpolate the atmosphere table 

986 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

987 pwv=atmCat[i]['pwv'], 

988 o3=atmCat[i]['o3'], 

989 tau=atmCat[i]['tau'], 

990 alpha=atmCat[i]['alpha'], 

991 zenith=zenith[i], 

992 ctranslamstd=[atmCat[i]['cTrans'], 

993 atmCat[i]['lamStd']]) 

994 else: 

995 # Run modtran 

996 modAtm = modGen(pmb=atmCat[i]['pmb'], 

997 pwv=atmCat[i]['pwv'], 

998 o3=atmCat[i]['o3'], 

999 tau=atmCat[i]['tau'], 

1000 alpha=atmCat[i]['alpha'], 

1001 zenith=zenith[i], 

1002 lambdaRange=lambdaRange, 

1003 lambdaStep=lambdaStep, 

1004 ctranslamstd=[atmCat[i]['cTrans'], 

1005 atmCat[i]['lamStd']]) 

1006 atmVals = modAtm['COMBINED'] 

1007 

1008 # Now need to create something to persist... 

1009 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

1010 wavelengths=atmLambda, 

1011 throughputAtMin=atmVals[0], 

1012 throughputAtMax=atmVals[-1]) 

1013 

1014 yield (int(visit), curve)