Coverage for python/lsst/fgcmcal/utilities.py: 9%

279 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-12 12:28 +0000

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Utility functions for fgcmcal. 

22 

23This file contains utility functions that are used by more than one task, 

24and do not need to be part of a task. 

25""" 

26 

27import numpy as np 

28import os 

29import re 

30 

31from lsst.daf.base import PropertyList 

32from lsst.daf.butler import Timespan 

33import lsst.afw.table as afwTable 

34import lsst.afw.image as afwImage 

35import lsst.afw.math as afwMath 

36import lsst.geom as geom 

37from lsst.obs.base import createInitialSkyWcs 

38 

39import fgcm 

40 

41 

42FGCM_EXP_FIELD = 'VISIT' 

43FGCM_CCD_FIELD = 'DETECTOR' 

44FGCM_ILLEGAL_VALUE = -9999.0 

45 

46 

47def makeConfigDict(config, log, camera, maxIter, 

48 resetFitParameters, outputZeropoints, 

49 lutFilterNames, tract=None): 

50 """ 

51 Make the FGCM fit cycle configuration dict 

52 

53 Parameters 

54 ---------- 

55 config: `lsst.fgcmcal.FgcmFitCycleConfig` 

56 Configuration object 

57 log: `lsst.log.Log` 

58 LSST log object 

59 camera: `lsst.afw.cameraGeom.Camera` 

60 Camera from the butler 

61 maxIter: `int` 

62 Maximum number of iterations 

63 resetFitParameters: `bool` 

64 Reset fit parameters before fitting? 

65 outputZeropoints: `bool` 

66 Compute zeropoints for output? 

67 lutFilterNames : array-like, `str` 

68 Array of physical filter names in the LUT. 

69 tract: `int`, optional 

70 Tract number for extending the output file name for debugging. 

71 Default is None. 

72 

73 Returns 

74 ------- 

75 configDict: `dict` 

76 Configuration dictionary for fgcm 

77 """ 

78 # Extract the bands that are _not_ being fit for fgcm configuration 

79 notFitBands = [b for b in config.bands if b not in config.fitBands] 

80 

81 # process the starColorCuts 

82 starColorCutList = [] 

83 for ccut in config.starColorCuts: 

84 if ccut == 'NO_DATA': 

85 # No color cuts to apply. 

86 break 

87 parts = ccut.split(',') 

88 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

89 

90 # process the refStarColorCuts 

91 refStarColorCutList = [] 

92 for ccut in config.refStarColorCuts: 

93 if ccut == 'NO_DATA': 

94 # No color cuts to apply. 

95 break 

96 parts = ccut.split(',') 

97 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

98 

99 # TODO: Having direct access to the mirror area from the camera would be 

100 # useful. See DM-16489. 

101 # Mirror area in cm**2 

102 if config.mirrorArea is None: 

103 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2. 

104 else: 

105 # Convert to square cm. 

106 mirrorArea = config.mirrorArea * 100.**2. 

107 

108 # Get approximate average camera gain: 

109 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()] 

110 cameraGain = float(np.median(gains)) 

111 

112 # Cut down the filter map to those that are in the LUT 

113 filterToBand = {filterName: config.physicalFilterMap[filterName] for 

114 filterName in lutFilterNames} 

115 

116 if tract is None: 

117 outfileBase = config.outfileBase 

118 else: 

119 outfileBase = '%s-%06d' % (config.outfileBase, tract) 

120 

121 # create a configuration dictionary for fgcmFitCycle 

122 configDict = {'outfileBase': outfileBase, 

123 'logger': log, 

124 'exposureFile': None, 

125 'obsFile': None, 

126 'indexFile': None, 

127 'lutFile': None, 

128 'mirrorArea': mirrorArea, 

129 'cameraGain': cameraGain, 

130 'ccdStartIndex': camera[0].getId(), 

131 'expField': FGCM_EXP_FIELD, 

132 'ccdField': FGCM_CCD_FIELD, 

133 'seeingField': 'DELTA_APER', 

134 'fwhmField': 'PSFSIGMA', 

135 'skyBrightnessField': 'SKYBACKGROUND', 

136 'deepFlag': 'DEEPFLAG', # unused 

137 'bands': list(config.bands), 

138 'fitBands': list(config.fitBands), 

139 'notFitBands': notFitBands, 

140 'requiredBands': list(config.requiredBands), 

141 'filterToBand': filterToBand, 

142 'logLevel': 'INFO', 

143 'nCore': config.nCore, 

144 'nStarPerRun': config.nStarPerRun, 

145 'nExpPerRun': config.nExpPerRun, 

146 'reserveFraction': config.reserveFraction, 

147 'freezeStdAtmosphere': config.freezeStdAtmosphere, 

148 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle, 

149 'superStarSubCCDDict': dict(config.superStarSubCcdDict), 

150 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder, 

151 'superStarSubCCDTriangular': config.superStarSubCcdTriangular, 

152 'superStarSigmaClip': config.superStarSigmaClip, 

153 'superStarPlotCCDResiduals': config.superStarPlotCcdResiduals, 

154 'focalPlaneSigmaClip': config.focalPlaneSigmaClip, 

155 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict), 

156 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder, 

157 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular, 

158 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict), 

159 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder, 

160 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd, 

161 'cycleNumber': config.cycleNumber, 

162 'maxIter': maxIter, 

163 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile, 

164 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd, 

165 'UTBoundary': config.utBoundary, 

166 'washMJDs': config.washMjds, 

167 'epochMJDs': config.epochMjds, 

168 'coatingMJDs': config.coatingMjds, 

169 'minObsPerBand': config.minObsPerBand, 

170 'latitude': config.latitude, 

171 'defaultCameraOrientation': config.defaultCameraOrientation, 

172 'brightObsGrayMax': config.brightObsGrayMax, 

173 'minStarPerCCD': config.minStarPerCcd, 

174 'minCCDPerExp': config.minCcdPerExp, 

175 'maxCCDGrayErr': config.maxCcdGrayErr, 

176 'minStarPerExp': config.minStarPerExp, 

177 'minExpPerNight': config.minExpPerNight, 

178 'expGrayInitialCut': config.expGrayInitialCut, 

179 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict), 

180 'expGrayHighCutDict': dict(config.expGrayHighCutDict), 

181 'expGrayRecoverCut': config.expGrayRecoverCut, 

182 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict), 

183 'expGrayErrRecoverCut': config.expGrayErrRecoverCut, 

184 'refStarSnMin': config.refStarSnMin, 

185 'refStarOutlierNSig': config.refStarOutlierNSig, 

186 'applyRefStarColorCuts': config.applyRefStarColorCuts, 

187 'useExposureReferenceOffset': config.useExposureReferenceOffset, 

188 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm. 

189 'starColorCuts': starColorCutList, 

190 'refStarColorCuts': refStarColorCutList, 

191 'aperCorrFitNBins': config.aperCorrFitNBins, 

192 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict), 

193 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'], 

194 'sedTermDict': config.sedterms.toDict()['data'], 

195 'colorSplitBands': list(config.colorSplitBands), 

196 'sigFgcmMaxErr': config.sigFgcmMaxErr, 

197 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict), 

198 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr, 

199 'approxThroughputDict': dict(config.approxThroughputDict), 

200 'sigmaCalRange': list(config.sigmaCalRange), 

201 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile), 

202 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile), 

203 'sigma0Phot': config.sigma0Phot, 

204 'mapLongitudeRef': config.mapLongitudeRef, 

205 'mapNSide': config.mapNSide, 

206 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet 

207 'varMinBand': 2, 

208 'useRetrievedPwv': False, 

209 'useNightlyRetrievedPwv': False, 

210 'pwvRetrievalSmoothBlock': 25, 

211 'useQuadraticPwv': config.useQuadraticPwv, 

212 'useRetrievedTauInit': False, 

213 'tauRetrievalMinCCDPerNight': 500, 

214 'modelMagErrors': config.modelMagErrors, 

215 'instrumentParsPerBand': config.instrumentParsPerBand, 

216 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT, 

217 'fitMirrorChromaticity': config.fitMirrorChromaticity, 

218 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict), 

219 'autoPhotometricCutNSig': config.autoPhotometricCutNSig, 

220 'autoHighCutNSig': config.autoHighCutNSig, 

221 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec, 

222 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec, 

223 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs, 

224 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx, 

225 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy, 

226 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside, 

227 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit, 

228 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar, 

229 'doComputeDeltaAperMap': config.doComputeDeltaAperMap, 

230 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd, 

231 'printOnly': False, 

232 'quietMode': config.quietMode, 

233 'randomSeed': config.randomSeed, 

234 'outputStars': False, 

235 'outputPath': os.path.abspath('.'), 

236 'clobber': True, 

237 'useSedLUT': False, 

238 'resetParameters': resetFitParameters, 

239 'doPlots': config.doPlots, 

240 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format 

241 'outputZeropoints': outputZeropoints} 

242 

243 return configDict 

244 

245 

246def translateFgcmLut(lutCat, physicalFilterMap): 

247 """ 

248 Translate the FGCM look-up-table into an fgcm-compatible object 

249 

250 Parameters 

251 ---------- 

252 lutCat: `lsst.afw.table.BaseCatalog` 

253 Catalog describing the FGCM look-up table 

254 physicalFilterMap: `dict` 

255 Physical filter to band mapping 

256 

257 Returns 

258 ------- 

259 fgcmLut: `lsst.fgcm.FgcmLut` 

260 Lookup table for FGCM 

261 lutIndexVals: `numpy.ndarray` 

262 Numpy array with LUT index information for FGCM 

263 lutStd: `numpy.ndarray` 

264 Numpy array with LUT standard throughput values for FGCM 

265 

266 Notes 

267 ----- 

268 After running this code, it is wise to `del lutCat` to clear the memory. 

269 """ 

270 

271 # first we need the lutIndexVals 

272 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U') 

273 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U') 

274 

275 # Note that any discrepancies between config values will raise relevant 

276 # exceptions in the FGCM code. 

277 

278 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str, 

279 lutFilterNames.size), 

280 ('STDFILTERNAMES', lutStdFilterNames.dtype.str, 

281 lutStdFilterNames.size), 

282 ('PMB', 'f8', lutCat[0]['pmb'].size), 

283 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size), 

284 ('PMBELEVATION', 'f8'), 

285 ('LAMBDANORM', 'f8'), 

286 ('PWV', 'f8', lutCat[0]['pwv'].size), 

287 ('O3', 'f8', lutCat[0]['o3'].size), 

288 ('TAU', 'f8', lutCat[0]['tau'].size), 

289 ('ALPHA', 'f8', lutCat[0]['alpha'].size), 

290 ('ZENITH', 'f8', lutCat[0]['zenith'].size), 

291 ('NCCD', 'i4')]) 

292 

293 lutIndexVals['FILTERNAMES'][:] = lutFilterNames 

294 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames 

295 lutIndexVals['PMB'][:] = lutCat[0]['pmb'] 

296 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor'] 

297 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation'] 

298 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm'] 

299 lutIndexVals['PWV'][:] = lutCat[0]['pwv'] 

300 lutIndexVals['O3'][:] = lutCat[0]['o3'] 

301 lutIndexVals['TAU'][:] = lutCat[0]['tau'] 

302 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha'] 

303 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith'] 

304 lutIndexVals['NCCD'] = lutCat[0]['nCcd'] 

305 

306 # now we need the Standard Values 

307 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'), 

308 ('PWVSTD', 'f8'), 

309 ('O3STD', 'f8'), 

310 ('TAUSTD', 'f8'), 

311 ('ALPHASTD', 'f8'), 

312 ('ZENITHSTD', 'f8'), 

313 ('LAMBDARANGE', 'f8', 2), 

314 ('LAMBDASTEP', 'f8'), 

315 ('LAMBDASTD', 'f8', lutFilterNames.size), 

316 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size), 

317 ('I0STD', 'f8', lutFilterNames.size), 

318 ('I1STD', 'f8', lutFilterNames.size), 

319 ('I10STD', 'f8', lutFilterNames.size), 

320 ('I2STD', 'f8', lutFilterNames.size), 

321 ('LAMBDAB', 'f8', lutFilterNames.size), 

322 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size), 

323 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)]) 

324 lutStd['PMBSTD'] = lutCat[0]['pmbStd'] 

325 lutStd['PWVSTD'] = lutCat[0]['pwvStd'] 

326 lutStd['O3STD'] = lutCat[0]['o3Std'] 

327 lutStd['TAUSTD'] = lutCat[0]['tauStd'] 

328 lutStd['ALPHASTD'] = lutCat[0]['alphaStd'] 

329 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd'] 

330 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:] 

331 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep'] 

332 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd'] 

333 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter'] 

334 lutStd['I0STD'][:] = lutCat[0]['i0Std'] 

335 lutStd['I1STD'][:] = lutCat[0]['i1Std'] 

336 lutStd['I10STD'][:] = lutCat[0]['i10Std'] 

337 lutStd['I2STD'][:] = lutCat[0]['i2Std'] 

338 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB'] 

339 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:] 

340 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:] 

341 

342 lutTypes = [row['luttype'] for row in lutCat] 

343 

344 # And the flattened look-up-table 

345 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'), 

346 ('I1', 'f4')]) 

347 

348 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:] 

349 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:] 

350 

351 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'), 

352 ('D_O3', 'f4'), 

353 ('D_LNTAU', 'f4'), 

354 ('D_ALPHA', 'f4'), 

355 ('D_SECZENITH', 'f4'), 

356 ('D_LNPWV_I1', 'f4'), 

357 ('D_O3_I1', 'f4'), 

358 ('D_LNTAU_I1', 'f4'), 

359 ('D_ALPHA_I1', 'f4'), 

360 ('D_SECZENITH_I1', 'f4')]) 

361 

362 for name in lutDerivFlat.dtype.names: 

363 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:] 

364 

365 # The fgcm.FgcmLUT() class copies all the LUT information into special 

366 # shared memory objects that will not blow up the memory usage when used 

367 # with python multiprocessing. Once all the numbers are copied, the 

368 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat) 

369 # will fall out of scope and can be cleaned up by the garbage collector. 

370 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd, 

371 filterToBand=physicalFilterMap) 

372 

373 return fgcmLut, lutIndexVals, lutStd 

374 

375 

376def translateVisitCatalog(visitCat): 

377 """ 

378 Translate the FGCM visit catalog to an fgcm-compatible object 

379 

380 Parameters 

381 ---------- 

382 visitCat: `lsst.afw.table.BaseCatalog` 

383 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask` 

384 

385 Returns 

386 ------- 

387 fgcmExpInfo: `numpy.ndarray` 

388 Numpy array for visit information for FGCM 

389 

390 Notes 

391 ----- 

392 After running this code, it is wise to `del visitCat` to clear the memory. 

393 """ 

394 

395 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'), 

396 ('MJD', 'f8'), 

397 ('EXPTIME', 'f8'), 

398 ('PSFSIGMA', 'f8'), 

399 ('DELTA_APER', 'f8'), 

400 ('SKYBACKGROUND', 'f8'), 

401 ('DEEPFLAG', 'i2'), 

402 ('TELHA', 'f8'), 

403 ('TELRA', 'f8'), 

404 ('TELDEC', 'f8'), 

405 ('TELROT', 'f8'), 

406 ('PMB', 'f8'), 

407 ('FILTERNAME', 'a50')]) 

408 fgcmExpInfo['VISIT'][:] = visitCat['visit'] 

409 fgcmExpInfo['MJD'][:] = visitCat['mjd'] 

410 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime'] 

411 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag'] 

412 fgcmExpInfo['TELHA'][:] = visitCat['telha'] 

413 fgcmExpInfo['TELRA'][:] = visitCat['telra'] 

414 fgcmExpInfo['TELDEC'][:] = visitCat['teldec'] 

415 fgcmExpInfo['TELROT'][:] = visitCat['telrot'] 

416 fgcmExpInfo['PMB'][:] = visitCat['pmb'] 

417 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma'] 

418 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper'] 

419 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground'] 

420 # Note that we have to go through asAstropy() to get a string 

421 # array out of an afwTable. This is faster than a row-by-row loop. 

422 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter'] 

423 

424 return fgcmExpInfo 

425 

426 

427def computeReferencePixelScale(camera): 

428 """ 

429 Compute the median pixel scale in the camera 

430 

431 Returns 

432 ------- 

433 pixelScale: `float` 

434 Average pixel scale (arcsecond) over the camera 

435 """ 

436 

437 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

438 orientation = 0.0*geom.degrees 

439 flipX = False 

440 

441 # Create a temporary visitInfo for input to createInitialSkyWcs 

442 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

443 boresightRotAngle=orientation, 

444 rotType=afwImage.RotType.SKY) 

445 

446 pixelScales = np.zeros(len(camera)) 

447 for i, detector in enumerate(camera): 

448 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

449 pixelScales[i] = wcs.getPixelScale().asArcseconds() 

450 

451 ok, = np.where(pixelScales > 0.0) 

452 return np.median(pixelScales[ok]) 

453 

454 

455def computeApproxPixelAreaFields(camera): 

456 """ 

457 Compute the approximate pixel area bounded fields from the camera 

458 geometry. 

459 

460 Parameters 

461 ---------- 

462 camera: `lsst.afw.cameraGeom.Camera` 

463 

464 Returns 

465 ------- 

466 approxPixelAreaFields: `dict` 

467 Dictionary of approximate area fields, keyed with detector ID 

468 """ 

469 

470 areaScaling = 1. / computeReferencePixelScale(camera)**2. 

471 

472 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem, 

473 # since we are looking for relative scales 

474 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

475 

476 flipX = False 

477 # Create a temporary visitInfo for input to createInitialSkyWcs 

478 # The orientation does not matter for the area computation 

479 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

480 boresightRotAngle=0.0*geom.degrees, 

481 rotType=afwImage.RotType.SKY) 

482 

483 approxPixelAreaFields = {} 

484 

485 for i, detector in enumerate(camera): 

486 key = detector.getId() 

487 

488 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

489 bbox = detector.getBBox() 

490 

491 areaField = afwMath.PixelAreaBoundedField(bbox, wcs, 

492 unit=geom.arcseconds, scaling=areaScaling) 

493 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField) 

494 

495 approxPixelAreaFields[key] = approxAreaField 

496 

497 return approxPixelAreaFields 

498 

499 

500def makeZptSchema(superStarChebyshevSize, zptChebyshevSize): 

501 """ 

502 Make the zeropoint schema 

503 

504 Parameters 

505 ---------- 

506 superStarChebyshevSize: `int` 

507 Length of the superstar chebyshev array 

508 zptChebyshevSize: `int` 

509 Length of the zeropoint chebyshev array 

510 

511 Returns 

512 ------- 

513 zptSchema: `lsst.afw.table.schema` 

514 """ 

515 

516 zptSchema = afwTable.Schema() 

517 

518 zptSchema.addField('visit', type=np.int64, doc='Visit number') 

519 zptSchema.addField('detector', type=np.int32, doc='Detector ID number') 

520 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: ' 

521 '1: Photometric, used in fit; ' 

522 '2: Photometric, not used in fit; ' 

523 '4: Non-photometric, on partly photometric night; ' 

524 '8: Non-photometric, on non-photometric night; ' 

525 '16: No zeropoint could be determined; ' 

526 '32: Too few stars for reliable gray computation')) 

527 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)') 

528 zptSchema.addField('fgcmZptErr', type=np.float64, 

529 doc='Error on zeropoint, estimated from repeatability + number of obs') 

530 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2, 

531 doc='maximum x/maximum y to scale to apply chebyshev parameters') 

532 zptSchema.addField('fgcmfZptCheb', type='ArrayD', 

533 size=zptChebyshevSize, 

534 doc='Chebyshev parameters (flattened) for zeropoint') 

535 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD', 

536 size=superStarChebyshevSize, 

537 doc='Chebyshev parameters (flattened) for superStarFlat') 

538 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband') 

539 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral') 

540 zptSchema.addField('fgcmR0', type=np.float64, 

541 doc='Retrieved i0 integral, estimated from stars (only for flag 1)') 

542 zptSchema.addField('fgcmR10', type=np.float64, 

543 doc='Retrieved i10 integral, estimated from stars (only for flag 1)') 

544 zptSchema.addField('fgcmGry', type=np.float64, 

545 doc='Estimated gray extinction relative to atmospheric solution; ' 

546 'only for fgcmFlag <= 4 (see fgcmFlag) ') 

547 zptSchema.addField('fgcmDeltaChrom', type=np.float64, 

548 doc='Mean chromatic correction for stars in this ccd; ' 

549 'only for fgcmFlag <= 4 (see fgcmFlag)') 

550 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd') 

551 zptSchema.addField('fgcmTilings', type=np.float64, 

552 doc='Number of photometric tilings used for solution for ccd') 

553 zptSchema.addField('fgcmFpGry', type=np.float64, 

554 doc='Average gray extinction over the full focal plane ' 

555 '(same for all ccds in a visit)') 

556 zptSchema.addField('fgcmFpGryBlue', type=np.float64, 

557 doc='Average gray extinction over the full focal plane ' 

558 'for 25% bluest stars') 

559 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64, 

560 doc='Error on Average gray extinction over the full focal plane ' 

561 'for 25% bluest stars') 

562 zptSchema.addField('fgcmFpGryRed', type=np.float64, 

563 doc='Average gray extinction over the full focal plane ' 

564 'for 25% reddest stars') 

565 zptSchema.addField('fgcmFpGryRedErr', type=np.float64, 

566 doc='Error on Average gray extinction over the full focal plane ' 

567 'for 25% reddest stars') 

568 zptSchema.addField('fgcmFpVar', type=np.float64, 

569 doc='Variance of gray extinction over the full focal plane ' 

570 '(same for all ccds in a visit)') 

571 zptSchema.addField('fgcmDust', type=np.float64, 

572 doc='Gray dust extinction from the primary/corrector' 

573 'at the time of the exposure') 

574 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction') 

575 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm') 

576 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64, 

577 doc=('Local background correction from brightest percentile ' 

578 '(value set by deltaMagBkgOffsetPercentile) calibration ' 

579 'stars.')) 

580 zptSchema.addField('exptime', type=np.float32, doc='Exposure time') 

581 zptSchema.addField('filtername', type=str, size=30, doc='Filter name') 

582 

583 return zptSchema 

584 

585 

586def makeZptCat(zptSchema, zpStruct): 

587 """ 

588 Make the zeropoint catalog for persistence 

589 

590 Parameters 

591 ---------- 

592 zptSchema: `lsst.afw.table.Schema` 

593 Zeropoint catalog schema 

594 zpStruct: `numpy.ndarray` 

595 Zeropoint structure from fgcm 

596 

597 Returns 

598 ------- 

599 zptCat: `afwTable.BaseCatalog` 

600 Zeropoint catalog for persistence 

601 """ 

602 

603 zptCat = afwTable.BaseCatalog(zptSchema) 

604 zptCat.reserve(zpStruct.size) 

605 

606 for filterName in zpStruct['FILTERNAME']: 

607 rec = zptCat.addNew() 

608 rec['filtername'] = filterName.decode('utf-8') 

609 

610 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD] 

611 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD] 

612 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG'] 

613 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT'] 

614 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR'] 

615 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX'] 

616 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB'] 

617 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB'] 

618 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0'] 

619 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10'] 

620 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0'] 

621 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10'] 

622 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY'] 

623 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM'] 

624 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR'] 

625 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS'] 

626 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY'] 

627 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0] 

628 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0] 

629 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2] 

630 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2] 

631 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR'] 

632 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST'] 

633 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT'] 

634 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR'] 

635 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG'] 

636 zptCat['exptime'][:] = zpStruct['EXPTIME'] 

637 

638 return zptCat 

639 

640 

641def makeAtmSchema(): 

642 """ 

643 Make the atmosphere schema 

644 

645 Returns 

646 ------- 

647 atmSchema: `lsst.afw.table.Schema` 

648 """ 

649 

650 atmSchema = afwTable.Schema() 

651 

652 atmSchema.addField('visit', type=np.int64, doc='Visit number') 

653 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)') 

654 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)') 

655 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth') 

656 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope') 

657 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)') 

658 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)') 

659 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor') 

660 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction') 

661 

662 return atmSchema 

663 

664 

665def makeAtmCat(atmSchema, atmStruct): 

666 """ 

667 Make the atmosphere catalog for persistence 

668 

669 Parameters 

670 ---------- 

671 atmSchema: `lsst.afw.table.Schema` 

672 Atmosphere catalog schema 

673 atmStruct: `numpy.ndarray` 

674 Atmosphere structure from fgcm 

675 

676 Returns 

677 ------- 

678 atmCat: `lsst.afw.table.BaseCatalog` 

679 Atmosphere catalog for persistence 

680 """ 

681 

682 atmCat = afwTable.BaseCatalog(atmSchema) 

683 atmCat.resize(atmStruct.size) 

684 

685 atmCat['visit'][:] = atmStruct['VISIT'] 

686 atmCat['pmb'][:] = atmStruct['PMB'] 

687 atmCat['pwv'][:] = atmStruct['PWV'] 

688 atmCat['tau'][:] = atmStruct['TAU'] 

689 atmCat['alpha'][:] = atmStruct['ALPHA'] 

690 atmCat['o3'][:] = atmStruct['O3'] 

691 atmCat['secZenith'][:] = atmStruct['SECZENITH'] 

692 atmCat['cTrans'][:] = atmStruct['CTRANS'] 

693 atmCat['lamStd'][:] = atmStruct['LAMSTD'] 

694 

695 return atmCat 

696 

697 

698def makeStdSchema(nBands): 

699 """ 

700 Make the standard star schema 

701 

702 Parameters 

703 ---------- 

704 nBands: `int` 

705 Number of bands in standard star catalog 

706 

707 Returns 

708 ------- 

709 stdSchema: `lsst.afw.table.Schema` 

710 """ 

711 

712 stdSchema = afwTable.SimpleTable.makeMinimalSchema() 

713 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations', 

714 size=nBands) 

715 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations', 

716 size=nBands) 

717 stdSchema.addField('mag_std_noabs', type='ArrayF', 

718 doc='Standard magnitude (no absolute calibration)', 

719 size=nBands) 

720 stdSchema.addField('magErr_std', type='ArrayF', 

721 doc='Standard magnitude error', 

722 size=nBands) 

723 stdSchema.addField('npsfcand', type='ArrayI', 

724 doc='Number of observations flagged as psf candidates', 

725 size=nBands) 

726 stdSchema.addField('delta_aper', type='ArrayF', 

727 doc='Delta mag (small - large aperture)', 

728 size=nBands) 

729 

730 return stdSchema 

731 

732 

733def makeStdCat(stdSchema, stdStruct, goodBands): 

734 """ 

735 Make the standard star catalog for persistence 

736 

737 Parameters 

738 ---------- 

739 stdSchema: `lsst.afw.table.Schema` 

740 Standard star catalog schema 

741 stdStruct: `numpy.ndarray` 

742 Standard star structure in FGCM format 

743 goodBands: `list` 

744 List of good band names used in stdStruct 

745 

746 Returns 

747 ------- 

748 stdCat: `lsst.afw.table.BaseCatalog` 

749 Standard star catalog for persistence 

750 """ 

751 

752 stdCat = afwTable.SimpleCatalog(stdSchema) 

753 stdCat.resize(stdStruct.size) 

754 

755 stdCat['id'][:] = stdStruct['FGCM_ID'] 

756 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees 

757 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees 

758 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :] 

759 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :] 

760 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :] 

761 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :] 

762 if 'NPSFCAND' in stdStruct.dtype.names: 

763 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :] 

764 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :] 

765 

766 md = PropertyList() 

767 md.set("BANDS", list(goodBands)) 

768 stdCat.setMetadata(md) 

769 

770 return stdCat 

771 

772 

773def computeApertureRadiusFromName(fluxField): 

774 """ 

775 Compute the radius associated with a CircularApertureFlux or ApFlux field. 

776 

777 Parameters 

778 ---------- 

779 fluxField : `str` 

780 CircularApertureFlux or ApFlux 

781 

782 Returns 

783 ------- 

784 apertureRadius : `float` 

785 Radius of the aperture field, in pixels. 

786 

787 Raises 

788 ------ 

789 RuntimeError: Raised if flux field is not a CircularApertureFlux, 

790 ApFlux, or apFlux. 

791 """ 

792 # TODO: Move this method to more general stack method in DM-25775 

793 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField) 

794 

795 if m is None: 

796 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux") 

797 

798 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10. 

799 

800 return apertureRadius 

801 

802 

803def extractReferenceMags(refStars, bands, filterMap): 

804 """ 

805 Extract reference magnitudes from refStars for given bands and 

806 associated filterMap. 

807 

808 Parameters 

809 ---------- 

810 refStars : `astropy.table.Table` or `lsst.afw.table.BaseCatalog` 

811 FGCM reference star catalog. 

812 bands : `list` 

813 List of bands for calibration. 

814 filterMap: `dict` 

815 FGCM mapping of filter to band. 

816 

817 Returns 

818 ------- 

819 refMag : `np.ndarray` 

820 nstar x nband array of reference magnitudes. 

821 refMagErr : `np.ndarray` 

822 nstar x nband array of reference magnitude errors. 

823 """ 

824 hasAstropyMeta = False 

825 try: 

826 meta = refStars.meta 

827 hasAstropyMeta = True 

828 except AttributeError: 

829 meta = refStars.getMetadata() 

830 

831 if 'FILTERNAMES' in meta: 

832 if hasAstropyMeta: 

833 filternames = meta['FILTERNAMES'] 

834 else: 

835 filternames = meta.getArray('FILTERNAMES') 

836 

837 # The reference catalog that fgcm wants has one entry per band 

838 # in the config file 

839 refMag = np.zeros((len(refStars), len(bands)), 

840 dtype=refStars['refMag'].dtype) + 99.0 

841 refMagErr = np.zeros_like(refMag) + 99.0 

842 for i, filtername in enumerate(filternames): 

843 # We are allowed to run the fit configured so that we do not 

844 # use every column in the reference catalog. 

845 try: 

846 band = filterMap[filtername] 

847 except KeyError: 

848 continue 

849 try: 

850 ind = bands.index(band) 

851 except ValueError: 

852 continue 

853 

854 refMag[:, ind] = refStars['refMag'][:, i] 

855 refMagErr[:, ind] = refStars['refMagErr'][:, i] 

856 else: 

857 raise RuntimeError("FGCM reference stars missing FILTERNAMES metadata.") 

858 

859 return refMag, refMagErr 

860 

861 

862def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

863 # For static calibrations, we search with a timespan that has unbounded 

864 # begin and end; we'll get an error if there's more than one match (because 

865 # then it's not static). 

866 timespan = Timespan(begin=None, end=None) 

867 result = [] 

868 # First iterate over all of the data IDs for this dataset type that are 

869 # consistent with the quantum data ID. 

870 for dataId in registry.queryDataIds(datasetType.dimensions, dataId=quantumDataId): 

871 # Find the dataset with this data ID using the unbounded timespan. 

872 if ref := registry.findDataset(datasetType, dataId, collections=collections, timespan=timespan): 

873 result.append(ref) 

874 return result