Coverage for python/lsst/fgcmcal/fgcmOutputProducts.py: 17%

340 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-20 10:24 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34import copy 

35 

36import numpy as np 

37import hpgeom as hpg 

38import esutil 

39from astropy import units 

40 

41import lsst.daf.base as dafBase 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45from lsst.afw.image import TransmissionCurve 

46from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

47from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

48from lsst.pipe.tasks.photoCal import PhotoCalTask 

49import lsst.geom 

50import lsst.afw.image as afwImage 

51import lsst.afw.math as afwMath 

52import lsst.afw.table as afwTable 

53from lsst.meas.algorithms import DatasetConfig 

54from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

55 

56from .utilities import computeApproxPixelAreaFields 

57from .utilities import lookupStaticCalibrations 

58from .utilities import FGCM_ILLEGAL_VALUE 

59 

60import fgcm 

61 

62__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask'] 

63 

64 

65class FgcmOutputProductsConnections(pipeBase.PipelineTaskConnections, 

66 dimensions=("instrument",), 

67 defaultTemplates={"cycleNumber": "0"}): 

68 camera = connectionTypes.PrerequisiteInput( 

69 doc="Camera instrument", 

70 name="camera", 

71 storageClass="Camera", 

72 dimensions=("instrument",), 

73 lookupFunction=lookupStaticCalibrations, 

74 isCalibration=True, 

75 ) 

76 

77 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

78 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

79 "chromatic corrections."), 

80 name="fgcmLookUpTable", 

81 storageClass="Catalog", 

82 dimensions=("instrument",), 

83 deferLoad=True, 

84 ) 

85 

86 fgcmVisitCatalog = connectionTypes.Input( 

87 doc="Catalog of visit information for fgcm", 

88 name="fgcmVisitCatalog", 

89 storageClass="Catalog", 

90 dimensions=("instrument",), 

91 deferLoad=True, 

92 ) 

93 

94 fgcmStandardStars = connectionTypes.Input( 

95 doc="Catalog of standard star data from fgcm fit", 

96 name="fgcmStandardStars{cycleNumber}", 

97 storageClass="SimpleCatalog", 

98 dimensions=("instrument",), 

99 deferLoad=True, 

100 ) 

101 

102 fgcmZeropoints = connectionTypes.Input( 

103 doc="Catalog of zeropoints from fgcm fit", 

104 name="fgcmZeropoints{cycleNumber}", 

105 storageClass="Catalog", 

106 dimensions=("instrument",), 

107 deferLoad=True, 

108 ) 

109 

110 fgcmAtmosphereParameters = connectionTypes.Input( 

111 doc="Catalog of atmosphere parameters from fgcm fit", 

112 name="fgcmAtmosphereParameters{cycleNumber}", 

113 storageClass="Catalog", 

114 dimensions=("instrument",), 

115 deferLoad=True, 

116 ) 

117 

118 refCat = connectionTypes.PrerequisiteInput( 

119 doc="Reference catalog to use for photometric calibration", 

120 name="cal_ref_cat", 

121 storageClass="SimpleCatalog", 

122 dimensions=("skypix",), 

123 deferLoad=True, 

124 multiple=True, 

125 ) 

126 

127 fgcmPhotoCalib = connectionTypes.Output( 

128 doc=("Per-visit photometric calibrations derived from fgcm calibration. " 

129 "These catalogs use detector id for the id and are sorted for " 

130 "fast lookups of a detector."), 

131 name="fgcmPhotoCalibCatalog", 

132 storageClass="ExposureCatalog", 

133 dimensions=("instrument", "visit",), 

134 multiple=True, 

135 ) 

136 

137 fgcmTransmissionAtmosphere = connectionTypes.Output( 

138 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

139 name="transmission_atmosphere_fgcm", 

140 storageClass="TransmissionCurve", 

141 dimensions=("instrument", 

142 "visit",), 

143 multiple=True, 

144 ) 

145 

146 fgcmOffsets = connectionTypes.Output( 

147 doc="Per-band offsets computed from doReferenceCalibration", 

148 name="fgcmReferenceCalibrationOffsets", 

149 storageClass="Catalog", 

150 dimensions=("instrument",), 

151 multiple=False, 

152 ) 

153 

154 def __init__(self, *, config=None): 

155 super().__init__(config=config) 

156 

157 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

158 raise ValueError("cycleNumber must be of integer format") 

159 

160 if not config.doReferenceCalibration: 

161 self.prerequisiteInputs.remove("refCat") 

162 if not config.doAtmosphereOutput: 

163 self.inputs.remove("fgcmAtmosphereParameters") 

164 if not config.doZeropointOutput: 

165 self.inputs.remove("fgcmZeropoints") 

166 if not config.doReferenceCalibration: 

167 self.outputs.remove("fgcmOffsets") 

168 

169 

170class FgcmOutputProductsConfig(pipeBase.PipelineTaskConfig, 

171 pipelineConnections=FgcmOutputProductsConnections): 

172 """Config for FgcmOutputProductsTask""" 

173 

174 cycleNumber = pexConfig.Field( 

175 doc="Final fit cycle from FGCM fit", 

176 dtype=int, 

177 default=0, 

178 deprecated=("This config is no longer used, and will be removed after v25. " 

179 "Please set config.connections.cycleNumber directly instead."), 

180 ) 

181 physicalFilterMap = pexConfig.DictField( 

182 doc="Mapping from 'physicalFilter' to band.", 

183 keytype=str, 

184 itemtype=str, 

185 default={}, 

186 ) 

187 # The following fields refer to calibrating from a reference 

188 # catalog, but in the future this might need to be expanded 

189 doReferenceCalibration = pexConfig.Field( 

190 doc=("Transfer 'absolute' calibration from reference catalog? " 

191 "This afterburner step is unnecessary if reference stars " 

192 "were used in the full fit in FgcmFitCycleTask."), 

193 dtype=bool, 

194 default=False, 

195 ) 

196 doRefcatOutput = pexConfig.Field( 

197 doc="Output standard stars in reference catalog format", 

198 dtype=bool, 

199 default=False, 

200 deprecated="doRefcatOutput is no longer supported; this config will be removed after v24" 

201 ) 

202 doAtmosphereOutput = pexConfig.Field( 

203 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

204 dtype=bool, 

205 default=True, 

206 ) 

207 doZeropointOutput = pexConfig.Field( 

208 doc="Output zeropoints in fgcm_photoCalib format", 

209 dtype=bool, 

210 default=True, 

211 ) 

212 doComposeWcsJacobian = pexConfig.Field( 

213 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

214 dtype=bool, 

215 default=True, 

216 ) 

217 doApplyMeanChromaticCorrection = pexConfig.Field( 

218 doc="Apply the mean chromatic correction to the zeropoints?", 

219 dtype=bool, 

220 default=True, 

221 ) 

222 refObjLoader = pexConfig.ConfigurableField( 

223 target=LoadIndexedReferenceObjectsTask, 

224 doc="reference object loader for 'absolute' photometric calibration", 

225 deprecated="refObjLoader is deprecated, and will be removed after v24", 

226 ) 

227 photoCal = pexConfig.ConfigurableField( 

228 target=PhotoCalTask, 

229 doc="task to perform 'absolute' calibration", 

230 ) 

231 referencePixelizationNside = pexConfig.Field( 

232 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

233 dtype=int, 

234 default=64, 

235 ) 

236 referencePixelizationMinStars = pexConfig.Field( 

237 doc=("Minimum number of stars per healpix pixel to select for comparison" 

238 "to the specified reference catalog"), 

239 dtype=int, 

240 default=200, 

241 ) 

242 referenceMinMatch = pexConfig.Field( 

243 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

244 dtype=int, 

245 default=50, 

246 ) 

247 referencePixelizationNPixels = pexConfig.Field( 

248 doc=("Number of healpix pixels to sample to do comparison. " 

249 "Doing too many will take a long time and not yield any more " 

250 "precise results because the final number is the median offset " 

251 "(per band) from the set of pixels."), 

252 dtype=int, 

253 default=100, 

254 ) 

255 datasetConfig = pexConfig.ConfigField( 

256 dtype=DatasetConfig, 

257 doc="Configuration for writing/reading ingested catalog", 

258 deprecated="The datasetConfig was only used for gen2; this config will be removed after v24.", 

259 ) 

260 

261 def setDefaults(self): 

262 pexConfig.Config.setDefaults(self) 

263 

264 # In order to transfer the "absolute" calibration from a reference 

265 # catalog to the relatively calibrated FGCM standard stars (one number 

266 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

267 # pixels. These basic settings ensure that only well-measured, good stars 

268 # from the source and reference catalogs are used for the matching. 

269 

270 # applyColorTerms needs to be False if doReferenceCalibration is False, 

271 # as is the new default after DM-16702 

272 self.photoCal.applyColorTerms = False 

273 self.photoCal.fluxField = 'instFlux' 

274 self.photoCal.magErrFloor = 0.003 

275 self.photoCal.match.referenceSelection.doSignalToNoise = True 

276 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

277 self.photoCal.match.sourceSelection.doSignalToNoise = True 

278 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

279 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

280 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

281 self.photoCal.match.sourceSelection.doFlags = True 

282 self.photoCal.match.sourceSelection.flags.good = [] 

283 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

284 self.photoCal.match.sourceSelection.doUnresolved = False 

285 

286 

287class FgcmOutputProductsTask(pipeBase.PipelineTask): 

288 """ 

289 Output products from FGCM global calibration. 

290 """ 

291 

292 ConfigClass = FgcmOutputProductsConfig 

293 _DefaultName = "fgcmOutputProducts" 

294 

295 def __init__(self, **kwargs): 

296 super().__init__(**kwargs) 

297 

298 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

299 handleDict = {} 

300 handleDict['camera'] = butlerQC.get(inputRefs.camera) 

301 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

302 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

303 handleDict['fgcmStandardStars'] = butlerQC.get(inputRefs.fgcmStandardStars) 

304 

305 if self.config.doZeropointOutput: 

306 handleDict['fgcmZeropoints'] = butlerQC.get(inputRefs.fgcmZeropoints) 

307 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

308 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

309 

310 if self.config.doAtmosphereOutput: 

311 handleDict['fgcmAtmosphereParameters'] = butlerQC.get(inputRefs.fgcmAtmosphereParameters) 

312 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

313 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

314 

315 if self.config.doReferenceCalibration: 

316 refConfig = LoadReferenceObjectsConfig() 

317 self.refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

318 for ref in inputRefs.refCat], 

319 refCats=butlerQC.get(inputRefs.refCat), 

320 log=self.log, 

321 config=refConfig) 

322 else: 

323 self.refObjLoader = None 

324 

325 struct = self.run(handleDict, self.config.physicalFilterMap) 

326 

327 # Output the photoCalib exposure catalogs 

328 if struct.photoCalibCatalogs is not None: 

329 self.log.info("Outputting photoCalib catalogs.") 

330 for visit, expCatalog in struct.photoCalibCatalogs: 

331 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

332 self.log.info("Done outputting photoCalib catalogs.") 

333 

334 # Output the atmospheres 

335 if struct.atmospheres is not None: 

336 self.log.info("Outputting atmosphere transmission files.") 

337 for visit, atm in struct.atmospheres: 

338 butlerQC.put(atm, atmRefDict[visit]) 

339 self.log.info("Done outputting atmosphere files.") 

340 

341 if self.config.doReferenceCalibration: 

342 # Turn offset into simple catalog for persistence if necessary 

343 schema = afwTable.Schema() 

344 schema.addField('offset', type=np.float64, 

345 doc="Post-process calibration offset (mag)") 

346 offsetCat = afwTable.BaseCatalog(schema) 

347 offsetCat.resize(len(struct.offsets)) 

348 offsetCat['offset'][:] = struct.offsets 

349 

350 butlerQC.put(offsetCat, outputRefs.fgcmOffsets) 

351 

352 return 

353 

354 def run(self, handleDict, physicalFilterMap): 

355 """Run the output products task. 

356 

357 Parameters 

358 ---------- 

359 handleDict : `dict` 

360 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

361 handle dictionary with keys: 

362 

363 ``"camera"`` 

364 Camera object (`lsst.afw.cameraGeom.Camera`) 

365 ``"fgcmLookUpTable"`` 

366 handle for the FGCM look-up table. 

367 ``"fgcmVisitCatalog"`` 

368 handle for visit summary catalog. 

369 ``"fgcmStandardStars"`` 

370 handle for the output standard star catalog. 

371 ``"fgcmZeropoints"`` 

372 handle for the zeropoint data catalog. 

373 ``"fgcmAtmosphereParameters"`` 

374 handle for the atmosphere parameter catalog. 

375 ``"fgcmBuildStarsTableConfig"`` 

376 Config for `lsst.fgcmcal.fgcmBuildStarsTableTask`. 

377 physicalFilterMap : `dict` 

378 Dictionary of mappings from physical filter to FGCM band. 

379 

380 Returns 

381 ------- 

382 retStruct : `lsst.pipe.base.Struct` 

383 Output structure with keys: 

384 

385 offsets : `np.ndarray` 

386 Final reference offsets, per band. 

387 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

388 Generator that returns (visit, transmissionCurve) tuples. 

389 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

390 Generator that returns (visit, exposureCatalog) tuples. 

391 """ 

392 stdCat = handleDict['fgcmStandardStars'].get() 

393 md = stdCat.getMetadata() 

394 bands = md.getArray('BANDS') 

395 

396 if self.config.doReferenceCalibration: 

397 lutCat = handleDict['fgcmLookUpTable'].get() 

398 offsets = self._computeReferenceOffsets(stdCat, lutCat, physicalFilterMap, bands) 

399 else: 

400 offsets = np.zeros(len(bands)) 

401 

402 del stdCat 

403 

404 if self.config.doZeropointOutput: 

405 zptCat = handleDict['fgcmZeropoints'].get() 

406 visitCat = handleDict['fgcmVisitCatalog'].get() 

407 

408 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands, 

409 physicalFilterMap) 

410 else: 

411 pcgen = None 

412 

413 if self.config.doAtmosphereOutput: 

414 atmCat = handleDict['fgcmAtmosphereParameters'].get() 

415 atmgen = self._outputAtmospheres(handleDict, atmCat) 

416 else: 

417 atmgen = None 

418 

419 retStruct = pipeBase.Struct(offsets=offsets, 

420 atmospheres=atmgen) 

421 retStruct.photoCalibCatalogs = pcgen 

422 

423 return retStruct 

424 

425 def generateTractOutputProducts(self, handleDict, tract, 

426 visitCat, zptCat, atmCat, stdCat, 

427 fgcmBuildStarsConfig): 

428 """ 

429 Generate the output products for a given tract, as specified in the config. 

430 

431 This method is here to have an alternate entry-point for 

432 FgcmCalibrateTract. 

433 

434 Parameters 

435 ---------- 

436 handleDict : `dict` 

437 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

438 handle dictionary with keys: 

439 

440 ``"camera"`` 

441 Camera object (`lsst.afw.cameraGeom.Camera`) 

442 ``"fgcmLookUpTable"`` 

443 handle for the FGCM look-up table. 

444 tract : `int` 

445 Tract number 

446 visitCat : `lsst.afw.table.BaseCatalog` 

447 FGCM visitCat from `FgcmBuildStarsTask` 

448 zptCat : `lsst.afw.table.BaseCatalog` 

449 FGCM zeropoint catalog from `FgcmFitCycleTask` 

450 atmCat : `lsst.afw.table.BaseCatalog` 

451 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

452 stdCat : `lsst.afw.table.SimpleCatalog` 

453 FGCM standard star catalog from `FgcmFitCycleTask` 

454 fgcmBuildStarsConfig : `lsst.fgcmcal.FgcmBuildStarsConfig` 

455 Configuration object from `FgcmBuildStarsTask` 

456 

457 Returns 

458 ------- 

459 retStruct : `lsst.pipe.base.Struct` 

460 Output structure with keys: 

461 

462 offsets : `np.ndarray` 

463 Final reference offsets, per band. 

464 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

465 Generator that returns (visit, transmissionCurve) tuples. 

466 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

467 Generator that returns (visit, exposureCatalog) tuples. 

468 """ 

469 physicalFilterMap = fgcmBuildStarsConfig.physicalFilterMap 

470 

471 md = stdCat.getMetadata() 

472 bands = md.getArray('BANDS') 

473 

474 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

475 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

476 "in fgcmBuildStarsTask.") 

477 

478 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

479 self.log.warning("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

480 

481 if self.config.doReferenceCalibration: 

482 lutCat = handleDict['fgcmLookUpTable'].get() 

483 offsets = self._computeReferenceOffsets(stdCat, lutCat, bands, physicalFilterMap) 

484 else: 

485 offsets = np.zeros(len(bands)) 

486 

487 if self.config.doZeropointOutput: 

488 pcgen = self._outputZeropoints(handleDict['camera'], zptCat, visitCat, offsets, bands, 

489 physicalFilterMap) 

490 else: 

491 pcgen = None 

492 

493 if self.config.doAtmosphereOutput: 

494 atmgen = self._outputAtmospheres(handleDict, atmCat) 

495 else: 

496 atmgen = None 

497 

498 retStruct = pipeBase.Struct(offsets=offsets, 

499 atmospheres=atmgen) 

500 retStruct.photoCalibCatalogs = pcgen 

501 

502 return retStruct 

503 

504 def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands): 

505 """ 

506 Compute offsets relative to a reference catalog. 

507 

508 This method splits the star catalog into healpix pixels 

509 and computes the calibration transfer for a sample of 

510 these pixels to approximate the 'absolute' calibration 

511 values (on for each band) to apply to transfer the 

512 absolute scale. 

513 

514 Parameters 

515 ---------- 

516 stdCat : `lsst.afw.table.SimpleCatalog` 

517 FGCM standard stars 

518 lutCat : `lsst.afw.table.SimpleCatalog` 

519 FGCM Look-up table 

520 physicalFilterMap : `dict` 

521 Dictionary of mappings from physical filter to FGCM band. 

522 bands : `list` [`str`] 

523 List of band names from FGCM output 

524 Returns 

525 ------- 

526 offsets : `numpy.array` of floats 

527 Per band zeropoint offsets 

528 """ 

529 

530 # Only use stars that are observed in all the bands that were actually used 

531 # This will ensure that we use the same healpix pixels for the absolute 

532 # calibration of each band. 

533 minObs = stdCat['ngood'].min(axis=1) 

534 

535 goodStars = (minObs >= 1) 

536 stdCat = stdCat[goodStars] 

537 

538 self.log.info("Found %d stars with at least 1 good observation in each band" % 

539 (len(stdCat))) 

540 

541 # Associate each band with the appropriate physicalFilter and make 

542 # filterLabels 

543 filterLabels = [] 

544 

545 lutPhysicalFilters = lutCat[0]['physicalFilters'].split(',') 

546 lutStdPhysicalFilters = lutCat[0]['stdPhysicalFilters'].split(',') 

547 physicalFilterMapBands = list(physicalFilterMap.values()) 

548 physicalFilterMapFilters = list(physicalFilterMap.keys()) 

549 for band in bands: 

550 # Find a physical filter associated from the band by doing 

551 # a reverse lookup on the physicalFilterMap dict 

552 physicalFilterMapIndex = physicalFilterMapBands.index(band) 

553 physicalFilter = physicalFilterMapFilters[physicalFilterMapIndex] 

554 # Find the appropriate fgcm standard physicalFilter 

555 lutPhysicalFilterIndex = lutPhysicalFilters.index(physicalFilter) 

556 stdPhysicalFilter = lutStdPhysicalFilters[lutPhysicalFilterIndex] 

557 filterLabels.append(afwImage.FilterLabel(band=band, 

558 physical=stdPhysicalFilter)) 

559 

560 # We have to make a table for each pixel with flux/fluxErr 

561 # This is a temporary table generated for input to the photoCal task. 

562 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

563 # have had chromatic corrections applied to get to the standard system 

564 # specified by the atmosphere/instrumental parameters), nor are they 

565 # in Jansky (since they don't have a proper absolute calibration: the overall 

566 # zeropoint is estimated from the telescope size, etc.) 

567 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

568 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

569 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

570 doc="instrumental flux (counts)") 

571 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

572 doc="instrumental flux error (counts)") 

573 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

574 type='Flag', 

575 doc="bad flag") 

576 

577 # Split up the stars 

578 # Note that there is an assumption here that the ra/dec coords stored 

579 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

580 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

581 ipring = hpg.angle_to_pixel( 

582 self.config.referencePixelizationNside, 

583 stdCat['coord_ra'], 

584 stdCat['coord_dec'], 

585 degrees=False, 

586 ) 

587 h, rev = esutil.stat.histogram(ipring, rev=True) 

588 

589 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

590 

591 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

592 (gdpix.size, 

593 self.config.referencePixelizationNside, 

594 self.config.referencePixelizationMinStars)) 

595 

596 if gdpix.size < self.config.referencePixelizationNPixels: 

597 self.log.warning("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

598 (gdpix.size, self.config.referencePixelizationNPixels)) 

599 else: 

600 # Sample out the pixels we want to use 

601 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

602 

603 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

604 ('nstar', 'i4', len(bands)), 

605 ('nmatch', 'i4', len(bands)), 

606 ('zp', 'f4', len(bands)), 

607 ('zpErr', 'f4', len(bands))]) 

608 results['hpix'] = ipring[rev[rev[gdpix]]] 

609 

610 # We need a boolean index to deal with catalogs... 

611 selected = np.zeros(len(stdCat), dtype=bool) 

612 

613 refFluxFields = [None]*len(bands) 

614 

615 for p_index, pix in enumerate(gdpix): 

616 i1a = rev[rev[pix]: rev[pix + 1]] 

617 

618 # the stdCat afwTable can only be indexed with boolean arrays, 

619 # and not numpy index arrays (see DM-16497). This little trick 

620 # converts the index array into a boolean array 

621 selected[:] = False 

622 selected[i1a] = True 

623 

624 for b_index, filterLabel in enumerate(filterLabels): 

625 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b_index, 

626 filterLabel, stdCat, 

627 selected, refFluxFields) 

628 results['nstar'][p_index, b_index] = len(i1a) 

629 results['nmatch'][p_index, b_index] = len(struct.arrays.refMag) 

630 results['zp'][p_index, b_index] = struct.zp 

631 results['zpErr'][p_index, b_index] = struct.sigma 

632 

633 # And compute the summary statistics 

634 offsets = np.zeros(len(bands)) 

635 

636 for b_index, band in enumerate(bands): 

637 # make configurable 

638 ok, = np.where(results['nmatch'][:, b_index] >= self.config.referenceMinMatch) 

639 offsets[b_index] = np.median(results['zp'][ok, b_index]) 

640 # use median absolute deviation to estimate Normal sigma 

641 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

642 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b_index] - offsets[b_index])) 

643 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f", 

644 band, offsets[b_index], madSigma) 

645 

646 return offsets 

647 

648 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

649 b_index, filterLabel, stdCat, selected, refFluxFields): 

650 """ 

651 Compute the zeropoint offset between the fgcm stdCat and the reference 

652 stars for one pixel in one band 

653 

654 Parameters 

655 ---------- 

656 sourceMapper : `lsst.afw.table.SchemaMapper` 

657 Mapper to go from stdCat to calibratable catalog 

658 badStarKey : `lsst.afw.table.Key` 

659 Key for the field with bad stars 

660 b_index : `int` 

661 Index of the band in the star catalog 

662 filterLabel : `lsst.afw.image.FilterLabel` 

663 filterLabel with band and physical filter 

664 stdCat : `lsst.afw.table.SimpleCatalog` 

665 FGCM standard stars 

666 selected : `numpy.array(dtype=bool)` 

667 Boolean array of which stars are in the pixel 

668 refFluxFields : `list` 

669 List of names of flux fields for reference catalog 

670 """ 

671 

672 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

673 sourceCat.reserve(selected.sum()) 

674 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

675 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b_index]/(-2.5)) 

676 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b_index] 

677 * sourceCat['instFlux']) 

678 # Make sure we only use stars that have valid measurements 

679 # (This is perhaps redundant with requirements above that the 

680 # stars be observed in all bands, but it can't hurt) 

681 badStar = (stdCat['mag_std_noabs'][selected, b_index] > 90.0) 

682 for rec in sourceCat[badStar]: 

683 rec.set(badStarKey, True) 

684 

685 exposure = afwImage.ExposureF() 

686 exposure.setFilter(filterLabel) 

687 

688 if refFluxFields[b_index] is None: 

689 # Need to find the flux field in the reference catalog 

690 # to work around limitations of DirectMatch in PhotoCal 

691 ctr = stdCat[0].getCoord() 

692 rad = 0.05*lsst.geom.degrees 

693 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, filterLabel.bandLabel) 

694 refFluxFields[b_index] = refDataTest.fluxField 

695 

696 # Make a copy of the config so that we can modify it 

697 calConfig = copy.copy(self.config.photoCal.value) 

698 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b_index] 

699 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b_index] + 'Err' 

700 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

701 config=calConfig, 

702 schema=sourceCat.getSchema()) 

703 

704 struct = calTask.run(exposure, sourceCat) 

705 

706 return struct 

707 

708 def _formatCatalog(self, fgcmStarCat, offsets, bands): 

709 """ 

710 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

711 

712 Parameters 

713 ---------- 

714 fgcmStarCat : `lsst.afw.Table.SimpleCatalog` 

715 SimpleCatalog as output by fgcmcal 

716 offsets : `list` with len(self.bands) entries 

717 Zeropoint offsets to apply 

718 bands : `list` [`str`] 

719 List of band names from FGCM output 

720 

721 Returns 

722 ------- 

723 formattedCat: `lsst.afw.table.SimpleCatalog` 

724 SimpleCatalog suitable for using as a reference catalog 

725 """ 

726 

727 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

728 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(bands, 

729 addCentroid=False, 

730 addIsResolved=True, 

731 coordErrDim=0) 

732 sourceMapper.addMinimalSchema(minSchema) 

733 for band in bands: 

734 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

735 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

736 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

737 

738 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

739 formattedCat.reserve(len(fgcmStarCat)) 

740 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

741 

742 # Note that we don't have to set `resolved` because the default is False 

743 

744 for b, band in enumerate(bands): 

745 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

746 # We want fluxes in nJy from calibrated AB magnitudes 

747 # (after applying offset). Updated after RFC-549 and RFC-575. 

748 flux = (mag*units.ABmag).to_value(units.nJy) 

749 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

750 

751 formattedCat['%s_flux' % (band)][:] = flux 

752 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

753 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

754 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

755 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

756 

757 addRefCatMetadata(formattedCat) 

758 

759 return formattedCat 

760 

761 def _outputZeropoints(self, camera, zptCat, visitCat, offsets, bands, 

762 physicalFilterMap, tract=None): 

763 """Output the zeropoints in fgcm_photoCalib format. 

764 

765 Parameters 

766 ---------- 

767 camera : `lsst.afw.cameraGeom.Camera` 

768 Camera from the butler. 

769 zptCat : `lsst.afw.table.BaseCatalog` 

770 FGCM zeropoint catalog from `FgcmFitCycleTask`. 

771 visitCat : `lsst.afw.table.BaseCatalog` 

772 FGCM visitCat from `FgcmBuildStarsTask`. 

773 offsets : `numpy.array` 

774 Float array of absolute calibration offsets, one for each filter. 

775 bands : `list` [`str`] 

776 List of band names from FGCM output. 

777 physicalFilterMap : `dict` 

778 Dictionary of mappings from physical filter to FGCM band. 

779 tract: `int`, optional 

780 Tract number to output. Default is None (global calibration) 

781 

782 Returns 

783 ------- 

784 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

785 Generator that returns (visit, exposureCatalog) tuples. 

786 """ 

787 # Select visit/ccds where we have a calibration 

788 # This includes ccds where we were able to interpolate from neighboring 

789 # ccds. 

790 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

791 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) 

792 & (zptCat['fgcmZptVar'] > 0.0) 

793 & (zptCat['fgcmZpt'] > FGCM_ILLEGAL_VALUE)) 

794 

795 # Log warnings for any visit which has no valid zeropoints 

796 badVisits = np.unique(zptCat['visit'][~selected]) 

797 goodVisits = np.unique(zptCat['visit'][selected]) 

798 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

799 for allBadVisit in allBadVisits: 

800 self.log.warning(f'No suitable photoCalib for visit {allBadVisit}') 

801 

802 # Get a mapping from filtername to the offsets 

803 offsetMapping = {} 

804 for f in physicalFilterMap: 

805 # Not every filter in the map will necesarily have a band. 

806 if physicalFilterMap[f] in bands: 

807 offsetMapping[f] = offsets[bands.index(physicalFilterMap[f])] 

808 

809 # Get a mapping from "ccd" to the ccd index used for the scaling 

810 ccdMapping = {} 

811 for ccdIndex, detector in enumerate(camera): 

812 ccdMapping[detector.getId()] = ccdIndex 

813 

814 # And a mapping to get the flat-field scaling values 

815 scalingMapping = {} 

816 for rec in visitCat: 

817 scalingMapping[rec['visit']] = rec['scaling'] 

818 

819 if self.config.doComposeWcsJacobian: 

820 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

821 

822 # The zptCat is sorted by visit, which is useful 

823 lastVisit = -1 

824 zptVisitCatalog = None 

825 

826 metadata = dafBase.PropertyList() 

827 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

828 metadata.add("COMMENT", "Only detectors with data have entries.") 

829 

830 for rec in zptCat[selected]: 

831 # Retrieve overall scaling 

832 scaling = scalingMapping[rec['visit']][ccdMapping[rec['detector']]] 

833 

834 # The postCalibrationOffset describe any zeropoint offsets 

835 # to apply after the fgcm calibration. The first part comes 

836 # from the reference catalog match (used in testing). The 

837 # second part comes from the mean chromatic correction 

838 # (if configured). 

839 postCalibrationOffset = offsetMapping[rec['filtername']] 

840 if self.config.doApplyMeanChromaticCorrection: 

841 postCalibrationOffset += rec['fgcmDeltaChrom'] 

842 

843 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

844 rec['fgcmfZptChebXyMax']) 

845 # Convert from FGCM AB to nJy 

846 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

847 rec['fgcmfZptChebXyMax'], 

848 offset=postCalibrationOffset, 

849 scaling=scaling) 

850 

851 if self.config.doComposeWcsJacobian: 

852 

853 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['detector']], 

854 fgcmSuperStarField, 

855 fgcmZptField]) 

856 else: 

857 # The photoCalib is just the product of the fgcmSuperStarField and the 

858 # fgcmZptField 

859 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

860 

861 # The "mean" calibration will be set to the center of the ccd for reference 

862 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

863 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

864 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

865 calibrationErr=calibErr, 

866 calibration=fgcmField, 

867 isConstant=False) 

868 

869 # Return full per-visit exposure catalogs 

870 if rec['visit'] != lastVisit: 

871 # This is a new visit. If the last visit was not -1, yield 

872 # the ExposureCatalog 

873 if lastVisit > -1: 

874 # ensure that the detectors are in sorted order, for fast lookups 

875 zptVisitCatalog.sort() 

876 yield (int(lastVisit), zptVisitCatalog) 

877 else: 

878 # We need to create a new schema 

879 zptExpCatSchema = afwTable.ExposureTable.makeMinimalSchema() 

880 zptExpCatSchema.addField('visit', type='L', doc='Visit number') 

881 

882 # And start a new one 

883 zptVisitCatalog = afwTable.ExposureCatalog(zptExpCatSchema) 

884 zptVisitCatalog.setMetadata(metadata) 

885 

886 lastVisit = int(rec['visit']) 

887 

888 catRecord = zptVisitCatalog.addNew() 

889 catRecord['id'] = int(rec['detector']) 

890 catRecord['visit'] = rec['visit'] 

891 catRecord.setPhotoCalib(photoCalib) 

892 

893 # Final output of last exposure catalog 

894 # ensure that the detectors are in sorted order, for fast lookups 

895 zptVisitCatalog.sort() 

896 yield (int(lastVisit), zptVisitCatalog) 

897 

898 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

899 """ 

900 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

901 and scaling. 

902 

903 Parameters 

904 ---------- 

905 coefficients: `numpy.array` 

906 Flattened array of chebyshev coefficients 

907 xyMax: `list` of length 2 

908 Maximum x and y of the chebyshev bounding box 

909 offset: `float`, optional 

910 Absolute calibration offset. Default is 0.0 

911 scaling: `float`, optional 

912 Flat scaling value from fgcmBuildStars. Default is 1.0 

913 

914 Returns 

915 ------- 

916 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

917 """ 

918 

919 orderPlus1 = int(np.sqrt(coefficients.size)) 

920 pars = np.zeros((orderPlus1, orderPlus1)) 

921 

922 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

923 lsst.geom.Point2I(*xyMax)) 

924 

925 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) 

926 * (10.**(offset/-2.5))*scaling) 

927 

928 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

929 

930 return boundedField 

931 

932 def _outputAtmospheres(self, handleDict, atmCat): 

933 """ 

934 Output the atmospheres. 

935 

936 Parameters 

937 ---------- 

938 handleDict : `dict` 

939 All data handles are `lsst.daf.butler.DeferredDatasetHandle` 

940 The handleDict has the follownig keys: 

941 

942 ``"fgcmLookUpTable"`` 

943 handle for the FGCM look-up table. 

944 atmCat : `lsst.afw.table.BaseCatalog` 

945 FGCM atmosphere parameter catalog from fgcmFitCycleTask. 

946 

947 Returns 

948 ------- 

949 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

950 Generator that returns (visit, transmissionCurve) tuples. 

951 """ 

952 # First, we need to grab the look-up table and key info 

953 lutCat = handleDict['fgcmLookUpTable'].get() 

954 

955 atmosphereTableName = lutCat[0]['tablename'] 

956 elevation = lutCat[0]['elevation'] 

957 atmLambda = lutCat[0]['atmLambda'] 

958 lutCat = None 

959 

960 # Make the atmosphere table if possible 

961 try: 

962 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

963 atmTable.loadTable() 

964 except IOError: 

965 atmTable = None 

966 

967 if atmTable is None: 

968 # Try to use MODTRAN instead 

969 try: 

970 modGen = fgcm.ModtranGenerator(elevation) 

971 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

972 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

973 except (ValueError, IOError) as e: 

974 raise RuntimeError("FGCM look-up-table generated with modtran, " 

975 "but modtran not configured to run.") from e 

976 

977 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

978 

979 for i, visit in enumerate(atmCat['visit']): 

980 if atmTable is not None: 

981 # Interpolate the atmosphere table 

982 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

983 pwv=atmCat[i]['pwv'], 

984 o3=atmCat[i]['o3'], 

985 tau=atmCat[i]['tau'], 

986 alpha=atmCat[i]['alpha'], 

987 zenith=zenith[i], 

988 ctranslamstd=[atmCat[i]['cTrans'], 

989 atmCat[i]['lamStd']]) 

990 else: 

991 # Run modtran 

992 modAtm = modGen(pmb=atmCat[i]['pmb'], 

993 pwv=atmCat[i]['pwv'], 

994 o3=atmCat[i]['o3'], 

995 tau=atmCat[i]['tau'], 

996 alpha=atmCat[i]['alpha'], 

997 zenith=zenith[i], 

998 lambdaRange=lambdaRange, 

999 lambdaStep=lambdaStep, 

1000 ctranslamstd=[atmCat[i]['cTrans'], 

1001 atmCat[i]['lamStd']]) 

1002 atmVals = modAtm['COMBINED'] 

1003 

1004 # Now need to create something to persist... 

1005 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

1006 wavelengths=atmLambda, 

1007 throughputAtMin=atmVals[0], 

1008 throughputAtMax=atmVals[-1]) 

1009 

1010 yield (int(visit), curve)