Coverage for python/lsst/fgcmcal/utilities.py: 9%

279 statements  

« prev     ^ index     » next       coverage.py v7.4.2, created at 2024-02-22 13:59 +0000

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Utility functions for fgcmcal. 

22 

23This file contains utility functions that are used by more than one task, 

24and do not need to be part of a task. 

25""" 

26 

27import numpy as np 

28import os 

29import re 

30 

31from lsst.daf.base import PropertyList 

32from lsst.daf.butler import Timespan 

33import lsst.afw.table as afwTable 

34import lsst.afw.image as afwImage 

35import lsst.afw.math as afwMath 

36import lsst.geom as geom 

37from lsst.obs.base import createInitialSkyWcs 

38 

39import fgcm 

40 

41 

42FGCM_EXP_FIELD = 'VISIT' 

43FGCM_CCD_FIELD = 'DETECTOR' 

44FGCM_ILLEGAL_VALUE = -9999.0 

45 

46 

47def makeConfigDict(config, log, camera, maxIter, 

48 resetFitParameters, outputZeropoints, 

49 lutFilterNames, tract=None): 

50 """ 

51 Make the FGCM fit cycle configuration dict 

52 

53 Parameters 

54 ---------- 

55 config: `lsst.fgcmcal.FgcmFitCycleConfig` 

56 Configuration object 

57 log: `lsst.log.Log` 

58 LSST log object 

59 camera: `lsst.afw.cameraGeom.Camera` 

60 Camera from the butler 

61 maxIter: `int` 

62 Maximum number of iterations 

63 resetFitParameters: `bool` 

64 Reset fit parameters before fitting? 

65 outputZeropoints: `bool` 

66 Compute zeropoints for output? 

67 lutFilterNames : array-like, `str` 

68 Array of physical filter names in the LUT. 

69 tract: `int`, optional 

70 Tract number for extending the output file name for debugging. 

71 Default is None. 

72 

73 Returns 

74 ------- 

75 configDict: `dict` 

76 Configuration dictionary for fgcm 

77 """ 

78 # Extract the bands that are _not_ being fit for fgcm configuration 

79 notFitBands = [b for b in config.bands if b not in config.fitBands] 

80 

81 # process the starColorCuts 

82 starColorCutList = [] 

83 for ccut in config.starColorCuts: 

84 if ccut == 'NO_DATA': 

85 # No color cuts to apply. 

86 break 

87 parts = ccut.split(',') 

88 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

89 

90 # process the refStarColorCuts 

91 refStarColorCutList = [] 

92 for ccut in config.refStarColorCuts: 

93 if ccut == 'NO_DATA': 

94 # No color cuts to apply. 

95 break 

96 parts = ccut.split(',') 

97 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

98 

99 # TODO: Having direct access to the mirror area from the camera would be 

100 # useful. See DM-16489. 

101 # Mirror area in cm**2 

102 if config.mirrorArea is None: 

103 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2. 

104 else: 

105 # Convert to square cm. 

106 mirrorArea = config.mirrorArea * 100.**2. 

107 

108 # Get approximate average camera gain: 

109 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()] 

110 cameraGain = float(np.median(gains)) 

111 

112 # Cut down the filter map to those that are in the LUT 

113 filterToBand = {filterName: config.physicalFilterMap[filterName] for 

114 filterName in lutFilterNames} 

115 

116 if tract is None: 

117 outfileBase = config.outfileBase 

118 else: 

119 outfileBase = '%s-%06d' % (config.outfileBase, tract) 

120 

121 # create a configuration dictionary for fgcmFitCycle 

122 configDict = {'outfileBase': outfileBase, 

123 'logger': log, 

124 'exposureFile': None, 

125 'obsFile': None, 

126 'indexFile': None, 

127 'lutFile': None, 

128 'mirrorArea': mirrorArea, 

129 'cameraGain': cameraGain, 

130 'ccdStartIndex': camera[0].getId(), 

131 'expField': FGCM_EXP_FIELD, 

132 'ccdField': FGCM_CCD_FIELD, 

133 'seeingField': 'DELTA_APER', 

134 'fwhmField': 'PSFSIGMA', 

135 'skyBrightnessField': 'SKYBACKGROUND', 

136 'deepFlag': 'DEEPFLAG', # unused 

137 'bands': list(config.bands), 

138 'fitBands': list(config.fitBands), 

139 'notFitBands': notFitBands, 

140 'requiredBands': list(config.requiredBands), 

141 'filterToBand': filterToBand, 

142 'logLevel': 'INFO', 

143 'nCore': config.nCore, 

144 'nStarPerRun': config.nStarPerRun, 

145 'nExpPerRun': config.nExpPerRun, 

146 'reserveFraction': config.reserveFraction, 

147 'freezeStdAtmosphere': config.freezeStdAtmosphere, 

148 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle, 

149 'superStarSubCCDDict': dict(config.superStarSubCcdDict), 

150 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder, 

151 'superStarSubCCDTriangular': config.superStarSubCcdTriangular, 

152 'superStarSigmaClip': config.superStarSigmaClip, 

153 'superStarPlotCCDResiduals': config.superStarPlotCcdResiduals, 

154 'focalPlaneSigmaClip': config.focalPlaneSigmaClip, 

155 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict), 

156 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder, 

157 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular, 

158 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict), 

159 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder, 

160 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd, 

161 'cycleNumber': config.cycleNumber, 

162 'maxIter': maxIter, 

163 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile, 

164 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd, 

165 'UTBoundary': config.utBoundary, 

166 'washMJDs': config.washMjds, 

167 'epochMJDs': config.epochMjds, 

168 'coatingMJDs': config.coatingMjds, 

169 'minObsPerBand': config.minObsPerBand, 

170 'latitude': config.latitude, 

171 'defaultCameraOrientation': config.defaultCameraOrientation, 

172 'brightObsGrayMax': config.brightObsGrayMax, 

173 'minStarPerCCD': config.minStarPerCcd, 

174 'minCCDPerExp': config.minCcdPerExp, 

175 'maxCCDGrayErr': config.maxCcdGrayErr, 

176 'minStarPerExp': config.minStarPerExp, 

177 'minExpPerNight': config.minExpPerNight, 

178 'expGrayInitialCut': config.expGrayInitialCut, 

179 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict), 

180 'expGrayHighCutDict': dict(config.expGrayHighCutDict), 

181 'expGrayRecoverCut': config.expGrayRecoverCut, 

182 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict), 

183 'expGrayErrRecoverCut': config.expGrayErrRecoverCut, 

184 'refStarSnMin': config.refStarSnMin, 

185 'refStarOutlierNSig': config.refStarOutlierNSig, 

186 'applyRefStarColorCuts': config.applyRefStarColorCuts, 

187 'useExposureReferenceOffset': config.useExposureReferenceOffset, 

188 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm. 

189 'starColorCuts': starColorCutList, 

190 'refStarColorCuts': refStarColorCutList, 

191 'aperCorrFitNBins': config.aperCorrFitNBins, 

192 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict), 

193 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'], 

194 'sedTermDict': config.sedterms.toDict()['data'], 

195 'colorSplitBands': list(config.colorSplitBands), 

196 'sigFgcmMaxErr': config.sigFgcmMaxErr, 

197 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict), 

198 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr, 

199 'approxThroughputDict': dict(config.approxThroughputDict), 

200 'sigmaCalRange': list(config.sigmaCalRange), 

201 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile), 

202 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile), 

203 'sigma0Phot': config.sigma0Phot, 

204 'mapLongitudeRef': config.mapLongitudeRef, 

205 'mapNSide': config.mapNSide, 

206 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet 

207 'varMinBand': 2, 

208 'useRetrievedPwv': False, 

209 'useNightlyRetrievedPwv': False, 

210 'pwvRetrievalSmoothBlock': 25, 

211 'useQuadraticPwv': config.useQuadraticPwv, 

212 'useRetrievedTauInit': False, 

213 'tauRetrievalMinCCDPerNight': 500, 

214 'modelMagErrors': config.modelMagErrors, 

215 'instrumentParsPerBand': config.instrumentParsPerBand, 

216 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT, 

217 'fitMirrorChromaticity': config.fitMirrorChromaticity, 

218 'fitCCDChromaticityDict': dict(config.fitCcdChromaticityDict), 

219 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict), 

220 'autoPhotometricCutNSig': config.autoPhotometricCutNSig, 

221 'autoHighCutNSig': config.autoHighCutNSig, 

222 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec, 

223 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec, 

224 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs, 

225 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx, 

226 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy, 

227 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside, 

228 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit, 

229 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar, 

230 'doComputeDeltaAperMap': config.doComputeDeltaAperMap, 

231 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd, 

232 'printOnly': False, 

233 'quietMode': config.quietMode, 

234 'randomSeed': config.randomSeed, 

235 'outputStars': False, 

236 'outputPath': os.path.abspath('.'), 

237 'clobber': True, 

238 'useSedLUT': False, 

239 'resetParameters': resetFitParameters, 

240 'doPlots': config.doPlots, 

241 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format 

242 'outputZeropoints': outputZeropoints} 

243 

244 return configDict 

245 

246 

247def translateFgcmLut(lutCat, physicalFilterMap): 

248 """ 

249 Translate the FGCM look-up-table into an fgcm-compatible object 

250 

251 Parameters 

252 ---------- 

253 lutCat: `lsst.afw.table.BaseCatalog` 

254 Catalog describing the FGCM look-up table 

255 physicalFilterMap: `dict` 

256 Physical filter to band mapping 

257 

258 Returns 

259 ------- 

260 fgcmLut: `lsst.fgcm.FgcmLut` 

261 Lookup table for FGCM 

262 lutIndexVals: `numpy.ndarray` 

263 Numpy array with LUT index information for FGCM 

264 lutStd: `numpy.ndarray` 

265 Numpy array with LUT standard throughput values for FGCM 

266 

267 Notes 

268 ----- 

269 After running this code, it is wise to `del lutCat` to clear the memory. 

270 """ 

271 

272 # first we need the lutIndexVals 

273 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U') 

274 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U') 

275 

276 # Note that any discrepancies between config values will raise relevant 

277 # exceptions in the FGCM code. 

278 

279 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str, 

280 lutFilterNames.size), 

281 ('STDFILTERNAMES', lutStdFilterNames.dtype.str, 

282 lutStdFilterNames.size), 

283 ('PMB', 'f8', lutCat[0]['pmb'].size), 

284 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size), 

285 ('PMBELEVATION', 'f8'), 

286 ('LAMBDANORM', 'f8'), 

287 ('PWV', 'f8', lutCat[0]['pwv'].size), 

288 ('O3', 'f8', lutCat[0]['o3'].size), 

289 ('TAU', 'f8', lutCat[0]['tau'].size), 

290 ('ALPHA', 'f8', lutCat[0]['alpha'].size), 

291 ('ZENITH', 'f8', lutCat[0]['zenith'].size), 

292 ('NCCD', 'i4')]) 

293 

294 lutIndexVals['FILTERNAMES'][:] = lutFilterNames 

295 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames 

296 lutIndexVals['PMB'][:] = lutCat[0]['pmb'] 

297 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor'] 

298 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation'] 

299 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm'] 

300 lutIndexVals['PWV'][:] = lutCat[0]['pwv'] 

301 lutIndexVals['O3'][:] = lutCat[0]['o3'] 

302 lutIndexVals['TAU'][:] = lutCat[0]['tau'] 

303 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha'] 

304 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith'] 

305 lutIndexVals['NCCD'] = lutCat[0]['nCcd'] 

306 

307 # now we need the Standard Values 

308 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'), 

309 ('PWVSTD', 'f8'), 

310 ('O3STD', 'f8'), 

311 ('TAUSTD', 'f8'), 

312 ('ALPHASTD', 'f8'), 

313 ('ZENITHSTD', 'f8'), 

314 ('LAMBDARANGE', 'f8', 2), 

315 ('LAMBDASTEP', 'f8'), 

316 ('LAMBDASTD', 'f8', lutFilterNames.size), 

317 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size), 

318 ('I0STD', 'f8', lutFilterNames.size), 

319 ('I1STD', 'f8', lutFilterNames.size), 

320 ('I10STD', 'f8', lutFilterNames.size), 

321 ('I2STD', 'f8', lutFilterNames.size), 

322 ('LAMBDAB', 'f8', lutFilterNames.size), 

323 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size), 

324 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)]) 

325 lutStd['PMBSTD'] = lutCat[0]['pmbStd'] 

326 lutStd['PWVSTD'] = lutCat[0]['pwvStd'] 

327 lutStd['O3STD'] = lutCat[0]['o3Std'] 

328 lutStd['TAUSTD'] = lutCat[0]['tauStd'] 

329 lutStd['ALPHASTD'] = lutCat[0]['alphaStd'] 

330 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd'] 

331 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:] 

332 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep'] 

333 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd'] 

334 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter'] 

335 lutStd['I0STD'][:] = lutCat[0]['i0Std'] 

336 lutStd['I1STD'][:] = lutCat[0]['i1Std'] 

337 lutStd['I10STD'][:] = lutCat[0]['i10Std'] 

338 lutStd['I2STD'][:] = lutCat[0]['i2Std'] 

339 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB'] 

340 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:] 

341 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:] 

342 

343 lutTypes = [row['luttype'] for row in lutCat] 

344 

345 # And the flattened look-up-table 

346 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'), 

347 ('I1', 'f4')]) 

348 

349 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:] 

350 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:] 

351 

352 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'), 

353 ('D_O3', 'f4'), 

354 ('D_LNTAU', 'f4'), 

355 ('D_ALPHA', 'f4'), 

356 ('D_SECZENITH', 'f4'), 

357 ('D_LNPWV_I1', 'f4'), 

358 ('D_O3_I1', 'f4'), 

359 ('D_LNTAU_I1', 'f4'), 

360 ('D_ALPHA_I1', 'f4'), 

361 ('D_SECZENITH_I1', 'f4')]) 

362 

363 for name in lutDerivFlat.dtype.names: 

364 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:] 

365 

366 # The fgcm.FgcmLUT() class copies all the LUT information into special 

367 # shared memory objects that will not blow up the memory usage when used 

368 # with python multiprocessing. Once all the numbers are copied, the 

369 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat) 

370 # will fall out of scope and can be cleaned up by the garbage collector. 

371 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd, 

372 filterToBand=physicalFilterMap) 

373 

374 return fgcmLut, lutIndexVals, lutStd 

375 

376 

377def translateVisitCatalog(visitCat): 

378 """ 

379 Translate the FGCM visit catalog to an fgcm-compatible object 

380 

381 Parameters 

382 ---------- 

383 visitCat: `lsst.afw.table.BaseCatalog` 

384 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask` 

385 

386 Returns 

387 ------- 

388 fgcmExpInfo: `numpy.ndarray` 

389 Numpy array for visit information for FGCM 

390 

391 Notes 

392 ----- 

393 After running this code, it is wise to `del visitCat` to clear the memory. 

394 """ 

395 

396 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'), 

397 ('MJD', 'f8'), 

398 ('EXPTIME', 'f8'), 

399 ('PSFSIGMA', 'f8'), 

400 ('DELTA_APER', 'f8'), 

401 ('SKYBACKGROUND', 'f8'), 

402 ('DEEPFLAG', 'i2'), 

403 ('TELHA', 'f8'), 

404 ('TELRA', 'f8'), 

405 ('TELDEC', 'f8'), 

406 ('TELROT', 'f8'), 

407 ('PMB', 'f8'), 

408 ('FILTERNAME', 'a50')]) 

409 fgcmExpInfo['VISIT'][:] = visitCat['visit'] 

410 fgcmExpInfo['MJD'][:] = visitCat['mjd'] 

411 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime'] 

412 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag'] 

413 fgcmExpInfo['TELHA'][:] = visitCat['telha'] 

414 fgcmExpInfo['TELRA'][:] = visitCat['telra'] 

415 fgcmExpInfo['TELDEC'][:] = visitCat['teldec'] 

416 fgcmExpInfo['TELROT'][:] = visitCat['telrot'] 

417 fgcmExpInfo['PMB'][:] = visitCat['pmb'] 

418 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma'] 

419 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper'] 

420 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground'] 

421 # Note that we have to go through asAstropy() to get a string 

422 # array out of an afwTable. This is faster than a row-by-row loop. 

423 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter'] 

424 

425 return fgcmExpInfo 

426 

427 

428def computeReferencePixelScale(camera): 

429 """ 

430 Compute the median pixel scale in the camera 

431 

432 Returns 

433 ------- 

434 pixelScale: `float` 

435 Average pixel scale (arcsecond) over the camera 

436 """ 

437 

438 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

439 orientation = 0.0*geom.degrees 

440 flipX = False 

441 

442 # Create a temporary visitInfo for input to createInitialSkyWcs 

443 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

444 boresightRotAngle=orientation, 

445 rotType=afwImage.RotType.SKY) 

446 

447 pixelScales = np.zeros(len(camera)) 

448 for i, detector in enumerate(camera): 

449 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

450 pixelScales[i] = wcs.getPixelScale().asArcseconds() 

451 

452 ok, = np.where(pixelScales > 0.0) 

453 return np.median(pixelScales[ok]) 

454 

455 

456def computeApproxPixelAreaFields(camera): 

457 """ 

458 Compute the approximate pixel area bounded fields from the camera 

459 geometry. 

460 

461 Parameters 

462 ---------- 

463 camera: `lsst.afw.cameraGeom.Camera` 

464 

465 Returns 

466 ------- 

467 approxPixelAreaFields: `dict` 

468 Dictionary of approximate area fields, keyed with detector ID 

469 """ 

470 

471 areaScaling = 1. / computeReferencePixelScale(camera)**2. 

472 

473 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem, 

474 # since we are looking for relative scales 

475 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

476 

477 flipX = False 

478 # Create a temporary visitInfo for input to createInitialSkyWcs 

479 # The orientation does not matter for the area computation 

480 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

481 boresightRotAngle=0.0*geom.degrees, 

482 rotType=afwImage.RotType.SKY) 

483 

484 approxPixelAreaFields = {} 

485 

486 for i, detector in enumerate(camera): 

487 key = detector.getId() 

488 

489 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

490 bbox = detector.getBBox() 

491 

492 areaField = afwMath.PixelAreaBoundedField(bbox, wcs, 

493 unit=geom.arcseconds, scaling=areaScaling) 

494 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField) 

495 

496 approxPixelAreaFields[key] = approxAreaField 

497 

498 return approxPixelAreaFields 

499 

500 

501def makeZptSchema(superStarChebyshevSize, zptChebyshevSize): 

502 """ 

503 Make the zeropoint schema 

504 

505 Parameters 

506 ---------- 

507 superStarChebyshevSize: `int` 

508 Length of the superstar chebyshev array 

509 zptChebyshevSize: `int` 

510 Length of the zeropoint chebyshev array 

511 

512 Returns 

513 ------- 

514 zptSchema: `lsst.afw.table.schema` 

515 """ 

516 

517 zptSchema = afwTable.Schema() 

518 

519 zptSchema.addField('visit', type=np.int64, doc='Visit number') 

520 zptSchema.addField('detector', type=np.int32, doc='Detector ID number') 

521 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: ' 

522 '1: Photometric, used in fit; ' 

523 '2: Photometric, not used in fit; ' 

524 '4: Non-photometric, on partly photometric night; ' 

525 '8: Non-photometric, on non-photometric night; ' 

526 '16: No zeropoint could be determined; ' 

527 '32: Too few stars for reliable gray computation')) 

528 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)') 

529 zptSchema.addField('fgcmZptErr', type=np.float64, 

530 doc='Error on zeropoint, estimated from repeatability + number of obs') 

531 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2, 

532 doc='maximum x/maximum y to scale to apply chebyshev parameters') 

533 zptSchema.addField('fgcmfZptCheb', type='ArrayD', 

534 size=zptChebyshevSize, 

535 doc='Chebyshev parameters (flattened) for zeropoint') 

536 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD', 

537 size=superStarChebyshevSize, 

538 doc='Chebyshev parameters (flattened) for superStarFlat') 

539 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband') 

540 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral') 

541 zptSchema.addField('fgcmR0', type=np.float64, 

542 doc='Retrieved i0 integral, estimated from stars (only for flag 1)') 

543 zptSchema.addField('fgcmR10', type=np.float64, 

544 doc='Retrieved i10 integral, estimated from stars (only for flag 1)') 

545 zptSchema.addField('fgcmGry', type=np.float64, 

546 doc='Estimated gray extinction relative to atmospheric solution; ' 

547 'only for fgcmFlag <= 4 (see fgcmFlag) ') 

548 zptSchema.addField('fgcmDeltaChrom', type=np.float64, 

549 doc='Mean chromatic correction for stars in this ccd; ' 

550 'only for fgcmFlag <= 4 (see fgcmFlag)') 

551 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd') 

552 zptSchema.addField('fgcmTilings', type=np.float64, 

553 doc='Number of photometric tilings used for solution for ccd') 

554 zptSchema.addField('fgcmFpGry', type=np.float64, 

555 doc='Average gray extinction over the full focal plane ' 

556 '(same for all ccds in a visit)') 

557 zptSchema.addField('fgcmFpGryBlue', type=np.float64, 

558 doc='Average gray extinction over the full focal plane ' 

559 'for 25% bluest stars') 

560 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64, 

561 doc='Error on Average gray extinction over the full focal plane ' 

562 'for 25% bluest stars') 

563 zptSchema.addField('fgcmFpGryRed', type=np.float64, 

564 doc='Average gray extinction over the full focal plane ' 

565 'for 25% reddest stars') 

566 zptSchema.addField('fgcmFpGryRedErr', type=np.float64, 

567 doc='Error on Average gray extinction over the full focal plane ' 

568 'for 25% reddest stars') 

569 zptSchema.addField('fgcmFpVar', type=np.float64, 

570 doc='Variance of gray extinction over the full focal plane ' 

571 '(same for all ccds in a visit)') 

572 zptSchema.addField('fgcmDust', type=np.float64, 

573 doc='Gray dust extinction from the primary/corrector' 

574 'at the time of the exposure') 

575 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction') 

576 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm') 

577 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64, 

578 doc=('Local background correction from brightest percentile ' 

579 '(value set by deltaMagBkgOffsetPercentile) calibration ' 

580 'stars.')) 

581 zptSchema.addField('exptime', type=np.float32, doc='Exposure time') 

582 zptSchema.addField('filtername', type=str, size=30, doc='Filter name') 

583 

584 return zptSchema 

585 

586 

587def makeZptCat(zptSchema, zpStruct): 

588 """ 

589 Make the zeropoint catalog for persistence 

590 

591 Parameters 

592 ---------- 

593 zptSchema: `lsst.afw.table.Schema` 

594 Zeropoint catalog schema 

595 zpStruct: `numpy.ndarray` 

596 Zeropoint structure from fgcm 

597 

598 Returns 

599 ------- 

600 zptCat: `afwTable.BaseCatalog` 

601 Zeropoint catalog for persistence 

602 """ 

603 

604 zptCat = afwTable.BaseCatalog(zptSchema) 

605 zptCat.reserve(zpStruct.size) 

606 

607 for filterName in zpStruct['FILTERNAME']: 

608 rec = zptCat.addNew() 

609 rec['filtername'] = filterName.decode('utf-8') 

610 

611 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD] 

612 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD] 

613 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG'] 

614 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT'] 

615 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR'] 

616 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX'] 

617 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB'] 

618 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB'] 

619 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0'] 

620 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10'] 

621 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0'] 

622 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10'] 

623 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY'] 

624 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM'] 

625 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR'] 

626 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS'] 

627 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY'] 

628 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0] 

629 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0] 

630 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2] 

631 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2] 

632 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR'] 

633 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST'] 

634 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT'] 

635 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR'] 

636 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG'] 

637 zptCat['exptime'][:] = zpStruct['EXPTIME'] 

638 

639 return zptCat 

640 

641 

642def makeAtmSchema(): 

643 """ 

644 Make the atmosphere schema 

645 

646 Returns 

647 ------- 

648 atmSchema: `lsst.afw.table.Schema` 

649 """ 

650 

651 atmSchema = afwTable.Schema() 

652 

653 atmSchema.addField('visit', type=np.int64, doc='Visit number') 

654 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)') 

655 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)') 

656 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth') 

657 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope') 

658 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)') 

659 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)') 

660 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor') 

661 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction') 

662 

663 return atmSchema 

664 

665 

666def makeAtmCat(atmSchema, atmStruct): 

667 """ 

668 Make the atmosphere catalog for persistence 

669 

670 Parameters 

671 ---------- 

672 atmSchema: `lsst.afw.table.Schema` 

673 Atmosphere catalog schema 

674 atmStruct: `numpy.ndarray` 

675 Atmosphere structure from fgcm 

676 

677 Returns 

678 ------- 

679 atmCat: `lsst.afw.table.BaseCatalog` 

680 Atmosphere catalog for persistence 

681 """ 

682 

683 atmCat = afwTable.BaseCatalog(atmSchema) 

684 atmCat.resize(atmStruct.size) 

685 

686 atmCat['visit'][:] = atmStruct['VISIT'] 

687 atmCat['pmb'][:] = atmStruct['PMB'] 

688 atmCat['pwv'][:] = atmStruct['PWV'] 

689 atmCat['tau'][:] = atmStruct['TAU'] 

690 atmCat['alpha'][:] = atmStruct['ALPHA'] 

691 atmCat['o3'][:] = atmStruct['O3'] 

692 atmCat['secZenith'][:] = atmStruct['SECZENITH'] 

693 atmCat['cTrans'][:] = atmStruct['CTRANS'] 

694 atmCat['lamStd'][:] = atmStruct['LAMSTD'] 

695 

696 return atmCat 

697 

698 

699def makeStdSchema(nBands): 

700 """ 

701 Make the standard star schema 

702 

703 Parameters 

704 ---------- 

705 nBands: `int` 

706 Number of bands in standard star catalog 

707 

708 Returns 

709 ------- 

710 stdSchema: `lsst.afw.table.Schema` 

711 """ 

712 

713 stdSchema = afwTable.SimpleTable.makeMinimalSchema() 

714 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations', 

715 size=nBands) 

716 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations', 

717 size=nBands) 

718 stdSchema.addField('mag_std_noabs', type='ArrayF', 

719 doc='Standard magnitude (no absolute calibration)', 

720 size=nBands) 

721 stdSchema.addField('magErr_std', type='ArrayF', 

722 doc='Standard magnitude error', 

723 size=nBands) 

724 stdSchema.addField('npsfcand', type='ArrayI', 

725 doc='Number of observations flagged as psf candidates', 

726 size=nBands) 

727 stdSchema.addField('delta_aper', type='ArrayF', 

728 doc='Delta mag (small - large aperture)', 

729 size=nBands) 

730 

731 return stdSchema 

732 

733 

734def makeStdCat(stdSchema, stdStruct, goodBands): 

735 """ 

736 Make the standard star catalog for persistence 

737 

738 Parameters 

739 ---------- 

740 stdSchema: `lsst.afw.table.Schema` 

741 Standard star catalog schema 

742 stdStruct: `numpy.ndarray` 

743 Standard star structure in FGCM format 

744 goodBands: `list` 

745 List of good band names used in stdStruct 

746 

747 Returns 

748 ------- 

749 stdCat: `lsst.afw.table.BaseCatalog` 

750 Standard star catalog for persistence 

751 """ 

752 

753 stdCat = afwTable.SimpleCatalog(stdSchema) 

754 stdCat.resize(stdStruct.size) 

755 

756 stdCat['id'][:] = stdStruct['FGCM_ID'] 

757 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees 

758 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees 

759 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :] 

760 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :] 

761 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :] 

762 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :] 

763 if 'NPSFCAND' in stdStruct.dtype.names: 

764 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :] 

765 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :] 

766 

767 md = PropertyList() 

768 md.set("BANDS", list(goodBands)) 

769 stdCat.setMetadata(md) 

770 

771 return stdCat 

772 

773 

774def computeApertureRadiusFromName(fluxField): 

775 """ 

776 Compute the radius associated with a CircularApertureFlux or ApFlux field. 

777 

778 Parameters 

779 ---------- 

780 fluxField : `str` 

781 CircularApertureFlux or ApFlux 

782 

783 Returns 

784 ------- 

785 apertureRadius : `float` 

786 Radius of the aperture field, in pixels. 

787 

788 Raises 

789 ------ 

790 RuntimeError: Raised if flux field is not a CircularApertureFlux, 

791 ApFlux, or apFlux. 

792 """ 

793 # TODO: Move this method to more general stack method in DM-25775 

794 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField) 

795 

796 if m is None: 

797 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux") 

798 

799 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10. 

800 

801 return apertureRadius 

802 

803 

804def extractReferenceMags(refStars, bands, filterMap): 

805 """ 

806 Extract reference magnitudes from refStars for given bands and 

807 associated filterMap. 

808 

809 Parameters 

810 ---------- 

811 refStars : `astropy.table.Table` or `lsst.afw.table.BaseCatalog` 

812 FGCM reference star catalog. 

813 bands : `list` 

814 List of bands for calibration. 

815 filterMap: `dict` 

816 FGCM mapping of filter to band. 

817 

818 Returns 

819 ------- 

820 refMag : `np.ndarray` 

821 nstar x nband array of reference magnitudes. 

822 refMagErr : `np.ndarray` 

823 nstar x nband array of reference magnitude errors. 

824 """ 

825 hasAstropyMeta = False 

826 try: 

827 meta = refStars.meta 

828 hasAstropyMeta = True 

829 except AttributeError: 

830 meta = refStars.getMetadata() 

831 

832 if 'FILTERNAMES' in meta: 

833 if hasAstropyMeta: 

834 filternames = meta['FILTERNAMES'] 

835 else: 

836 filternames = meta.getArray('FILTERNAMES') 

837 

838 # The reference catalog that fgcm wants has one entry per band 

839 # in the config file 

840 refMag = np.zeros((len(refStars), len(bands)), 

841 dtype=refStars['refMag'].dtype) + 99.0 

842 refMagErr = np.zeros_like(refMag) + 99.0 

843 for i, filtername in enumerate(filternames): 

844 # We are allowed to run the fit configured so that we do not 

845 # use every column in the reference catalog. 

846 try: 

847 band = filterMap[filtername] 

848 except KeyError: 

849 continue 

850 try: 

851 ind = bands.index(band) 

852 except ValueError: 

853 continue 

854 

855 refMag[:, ind] = refStars['refMag'][:, i] 

856 refMagErr[:, ind] = refStars['refMagErr'][:, i] 

857 else: 

858 raise RuntimeError("FGCM reference stars missing FILTERNAMES metadata.") 

859 

860 return refMag, refMagErr 

861 

862 

863def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

864 # For static calibrations, we search with a timespan that has unbounded 

865 # begin and end; we'll get an error if there's more than one match (because 

866 # then it's not static). 

867 timespan = Timespan(begin=None, end=None) 

868 result = [] 

869 # First iterate over all of the data IDs for this dataset type that are 

870 # consistent with the quantum data ID. 

871 for dataId in registry.queryDataIds(datasetType.dimensions, dataId=quantumDataId): 

872 # Find the dataset with this data ID using the unbounded timespan. 

873 if ref := registry.findDataset(datasetType, dataId, collections=collections, timespan=timespan): 

874 result.append(ref) 

875 return result