Coverage for python/lsst/fgcmcal/utilities.py: 10%

292 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-01 02:26 -0700

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Utility functions for fgcmcal. 

22 

23This file contains utility functions that are used by more than one task, 

24and do not need to be part of a task. 

25""" 

26 

27import numpy as np 

28import os 

29import re 

30from deprecated.sphinx import deprecated 

31 

32from lsst.daf.base import PropertyList 

33import lsst.afw.cameraGeom as afwCameraGeom 

34import lsst.afw.table as afwTable 

35import lsst.afw.image as afwImage 

36import lsst.afw.math as afwMath 

37import lsst.geom as geom 

38from lsst.obs.base import createInitialSkyWcs 

39from lsst.pipe.base import Instrument 

40 

41import fgcm 

42 

43 

44FGCM_EXP_FIELD = 'VISIT' 

45FGCM_CCD_FIELD = 'DETECTOR' 

46FGCM_ILLEGAL_VALUE = -9999.0 

47 

48 

49def makeConfigDict(config, log, camera, maxIter, 

50 resetFitParameters, outputZeropoints, 

51 lutFilterNames, tract=None): 

52 """ 

53 Make the FGCM fit cycle configuration dict 

54 

55 Parameters 

56 ---------- 

57 config: `lsst.fgcmcal.FgcmFitCycleConfig` 

58 Configuration object 

59 log: `lsst.log.Log` 

60 LSST log object 

61 camera: `lsst.afw.cameraGeom.Camera` 

62 Camera from the butler 

63 maxIter: `int` 

64 Maximum number of iterations 

65 resetFitParameters: `bool` 

66 Reset fit parameters before fitting? 

67 outputZeropoints: `bool` 

68 Compute zeropoints for output? 

69 lutFilterNames : array-like, `str` 

70 Array of physical filter names in the LUT. 

71 tract: `int`, optional 

72 Tract number for extending the output file name for debugging. 

73 Default is None. 

74 

75 Returns 

76 ------- 

77 configDict: `dict` 

78 Configuration dictionary for fgcm 

79 """ 

80 # Extract the bands that are _not_ being fit for fgcm configuration 

81 notFitBands = [b for b in config.bands if b not in config.fitBands] 

82 

83 # process the starColorCuts 

84 starColorCutList = [] 

85 for ccut in config.starColorCuts: 

86 if ccut == 'NO_DATA': 

87 # No color cuts to apply. 

88 break 

89 parts = ccut.split(',') 

90 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

91 

92 # process the refStarColorCuts 

93 refStarColorCutList = [] 

94 for ccut in config.refStarColorCuts: 

95 if ccut == 'NO_DATA': 

96 # No color cuts to apply. 

97 break 

98 parts = ccut.split(',') 

99 refStarColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

100 

101 # TODO: Having direct access to the mirror area from the camera would be 

102 # useful. See DM-16489. 

103 # Mirror area in cm**2 

104 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2. 

105 

106 # Get approximate average camera gain: 

107 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()] 

108 cameraGain = float(np.median(gains)) 

109 

110 # Cut down the filter map to those that are in the LUT 

111 filterToBand = {filterName: config.physicalFilterMap[filterName] for 

112 filterName in lutFilterNames} 

113 

114 if tract is None: 

115 outfileBase = config.outfileBase 

116 else: 

117 outfileBase = '%s-%06d' % (config.outfileBase, tract) 

118 

119 # create a configuration dictionary for fgcmFitCycle 

120 configDict = {'outfileBase': outfileBase, 

121 'logger': log, 

122 'exposureFile': None, 

123 'obsFile': None, 

124 'indexFile': None, 

125 'lutFile': None, 

126 'mirrorArea': mirrorArea, 

127 'cameraGain': cameraGain, 

128 'ccdStartIndex': camera[0].getId(), 

129 'expField': FGCM_EXP_FIELD, 

130 'ccdField': FGCM_CCD_FIELD, 

131 'seeingField': 'DELTA_APER', 

132 'fwhmField': 'PSFSIGMA', 

133 'skyBrightnessField': 'SKYBACKGROUND', 

134 'deepFlag': 'DEEPFLAG', # unused 

135 'bands': list(config.bands), 

136 'fitBands': list(config.fitBands), 

137 'notFitBands': notFitBands, 

138 'requiredBands': list(config.requiredBands), 

139 'filterToBand': filterToBand, 

140 'logLevel': 'INFO', 

141 'nCore': config.nCore, 

142 'nStarPerRun': config.nStarPerRun, 

143 'nExpPerRun': config.nExpPerRun, 

144 'reserveFraction': config.reserveFraction, 

145 'freezeStdAtmosphere': config.freezeStdAtmosphere, 

146 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle, 

147 'superStarSubCCDDict': dict(config.superStarSubCcdDict), 

148 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder, 

149 'superStarSubCCDTriangular': config.superStarSubCcdTriangular, 

150 'superStarSigmaClip': config.superStarSigmaClip, 

151 'focalPlaneSigmaClip': config.focalPlaneSigmaClip, 

152 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict), 

153 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder, 

154 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular, 

155 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict), 

156 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder, 

157 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd, 

158 'cycleNumber': config.cycleNumber, 

159 'maxIter': maxIter, 

160 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile, 

161 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd, 

162 'UTBoundary': config.utBoundary, 

163 'washMJDs': config.washMjds, 

164 'epochMJDs': config.epochMjds, 

165 'coatingMJDs': config.coatingMjds, 

166 'minObsPerBand': config.minObsPerBand, 

167 'latitude': config.latitude, 

168 'defaultCameraOrientation': config.defaultCameraOrientation, 

169 'brightObsGrayMax': config.brightObsGrayMax, 

170 'minStarPerCCD': config.minStarPerCcd, 

171 'minCCDPerExp': config.minCcdPerExp, 

172 'maxCCDGrayErr': config.maxCcdGrayErr, 

173 'minStarPerExp': config.minStarPerExp, 

174 'minExpPerNight': config.minExpPerNight, 

175 'expGrayInitialCut': config.expGrayInitialCut, 

176 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict), 

177 'expGrayHighCutDict': dict(config.expGrayHighCutDict), 

178 'expGrayRecoverCut': config.expGrayRecoverCut, 

179 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict), 

180 'expGrayErrRecoverCut': config.expGrayErrRecoverCut, 

181 'refStarSnMin': config.refStarSnMin, 

182 'refStarOutlierNSig': config.refStarOutlierNSig, 

183 'applyRefStarColorCuts': config.applyRefStarColorCuts, 

184 'useExposureReferenceOffset': config.useExposureReferenceOffset, 

185 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm. 

186 'starColorCuts': starColorCutList, 

187 'refStarColorCuts': refStarColorCutList, 

188 'aperCorrFitNBins': config.aperCorrFitNBins, 

189 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict), 

190 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'], 

191 'sedTermDict': config.sedterms.toDict()['data'], 

192 'colorSplitBands': list(config.colorSplitBands), 

193 'sigFgcmMaxErr': config.sigFgcmMaxErr, 

194 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict), 

195 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr, 

196 'approxThroughputDict': dict(config.approxThroughputDict), 

197 'sigmaCalRange': list(config.sigmaCalRange), 

198 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile), 

199 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile), 

200 'sigma0Phot': config.sigma0Phot, 

201 'mapLongitudeRef': config.mapLongitudeRef, 

202 'mapNSide': config.mapNSide, 

203 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet 

204 'varMinBand': 2, 

205 'useRetrievedPwv': False, 

206 'useNightlyRetrievedPwv': False, 

207 'pwvRetrievalSmoothBlock': 25, 

208 'useQuadraticPwv': config.useQuadraticPwv, 

209 'useRetrievedTauInit': False, 

210 'tauRetrievalMinCCDPerNight': 500, 

211 'modelMagErrors': config.modelMagErrors, 

212 'instrumentParsPerBand': config.instrumentParsPerBand, 

213 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT, 

214 'fitMirrorChromaticity': config.fitMirrorChromaticity, 

215 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict), 

216 'autoPhotometricCutNSig': config.autoPhotometricCutNSig, 

217 'autoHighCutNSig': config.autoHighCutNSig, 

218 'deltaAperInnerRadiusArcsec': config.deltaAperInnerRadiusArcsec, 

219 'deltaAperOuterRadiusArcsec': config.deltaAperOuterRadiusArcsec, 

220 'deltaAperFitMinNgoodObs': config.deltaAperFitMinNgoodObs, 

221 'deltaAperFitPerCcdNx': config.deltaAperFitPerCcdNx, 

222 'deltaAperFitPerCcdNy': config.deltaAperFitPerCcdNy, 

223 'deltaAperFitSpatialNside': config.deltaAperFitSpatialNside, 

224 'doComputeDeltaAperExposures': config.doComputeDeltaAperPerVisit, 

225 'doComputeDeltaAperStars': config.doComputeDeltaAperPerStar, 

226 'doComputeDeltaAperMap': config.doComputeDeltaAperMap, 

227 'doComputeDeltaAperPerCcd': config.doComputeDeltaAperPerCcd, 

228 'printOnly': False, 

229 'quietMode': config.quietMode, 

230 'randomSeed': config.randomSeed, 

231 'outputStars': False, 

232 'outputPath': os.path.abspath('.'), 

233 'clobber': True, 

234 'useSedLUT': False, 

235 'resetParameters': resetFitParameters, 

236 'doPlots': config.doPlots, 

237 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format 

238 'outputZeropoints': outputZeropoints} 

239 

240 return configDict 

241 

242 

243def translateFgcmLut(lutCat, physicalFilterMap): 

244 """ 

245 Translate the FGCM look-up-table into an fgcm-compatible object 

246 

247 Parameters 

248 ---------- 

249 lutCat: `lsst.afw.table.BaseCatalog` 

250 Catalog describing the FGCM look-up table 

251 physicalFilterMap: `dict` 

252 Physical filter to band mapping 

253 

254 Returns 

255 ------- 

256 fgcmLut: `lsst.fgcm.FgcmLut` 

257 Lookup table for FGCM 

258 lutIndexVals: `numpy.ndarray` 

259 Numpy array with LUT index information for FGCM 

260 lutStd: `numpy.ndarray` 

261 Numpy array with LUT standard throughput values for FGCM 

262 

263 Notes 

264 ----- 

265 After running this code, it is wise to `del lutCat` to clear the memory. 

266 """ 

267 

268 # first we need the lutIndexVals 

269 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U') 

270 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U') 

271 

272 # Note that any discrepancies between config values will raise relevant 

273 # exceptions in the FGCM code. 

274 

275 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str, 

276 lutFilterNames.size), 

277 ('STDFILTERNAMES', lutStdFilterNames.dtype.str, 

278 lutStdFilterNames.size), 

279 ('PMB', 'f8', lutCat[0]['pmb'].size), 

280 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size), 

281 ('PMBELEVATION', 'f8'), 

282 ('LAMBDANORM', 'f8'), 

283 ('PWV', 'f8', lutCat[0]['pwv'].size), 

284 ('O3', 'f8', lutCat[0]['o3'].size), 

285 ('TAU', 'f8', lutCat[0]['tau'].size), 

286 ('ALPHA', 'f8', lutCat[0]['alpha'].size), 

287 ('ZENITH', 'f8', lutCat[0]['zenith'].size), 

288 ('NCCD', 'i4')]) 

289 

290 lutIndexVals['FILTERNAMES'][:] = lutFilterNames 

291 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames 

292 lutIndexVals['PMB'][:] = lutCat[0]['pmb'] 

293 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor'] 

294 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation'] 

295 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm'] 

296 lutIndexVals['PWV'][:] = lutCat[0]['pwv'] 

297 lutIndexVals['O3'][:] = lutCat[0]['o3'] 

298 lutIndexVals['TAU'][:] = lutCat[0]['tau'] 

299 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha'] 

300 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith'] 

301 lutIndexVals['NCCD'] = lutCat[0]['nCcd'] 

302 

303 # now we need the Standard Values 

304 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'), 

305 ('PWVSTD', 'f8'), 

306 ('O3STD', 'f8'), 

307 ('TAUSTD', 'f8'), 

308 ('ALPHASTD', 'f8'), 

309 ('ZENITHSTD', 'f8'), 

310 ('LAMBDARANGE', 'f8', 2), 

311 ('LAMBDASTEP', 'f8'), 

312 ('LAMBDASTD', 'f8', lutFilterNames.size), 

313 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size), 

314 ('I0STD', 'f8', lutFilterNames.size), 

315 ('I1STD', 'f8', lutFilterNames.size), 

316 ('I10STD', 'f8', lutFilterNames.size), 

317 ('I2STD', 'f8', lutFilterNames.size), 

318 ('LAMBDAB', 'f8', lutFilterNames.size), 

319 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size), 

320 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)]) 

321 lutStd['PMBSTD'] = lutCat[0]['pmbStd'] 

322 lutStd['PWVSTD'] = lutCat[0]['pwvStd'] 

323 lutStd['O3STD'] = lutCat[0]['o3Std'] 

324 lutStd['TAUSTD'] = lutCat[0]['tauStd'] 

325 lutStd['ALPHASTD'] = lutCat[0]['alphaStd'] 

326 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd'] 

327 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:] 

328 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep'] 

329 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd'] 

330 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter'] 

331 lutStd['I0STD'][:] = lutCat[0]['i0Std'] 

332 lutStd['I1STD'][:] = lutCat[0]['i1Std'] 

333 lutStd['I10STD'][:] = lutCat[0]['i10Std'] 

334 lutStd['I2STD'][:] = lutCat[0]['i2Std'] 

335 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB'] 

336 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:] 

337 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:] 

338 

339 lutTypes = [row['luttype'] for row in lutCat] 

340 

341 # And the flattened look-up-table 

342 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'), 

343 ('I1', 'f4')]) 

344 

345 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:] 

346 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:] 

347 

348 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'), 

349 ('D_O3', 'f4'), 

350 ('D_LNTAU', 'f4'), 

351 ('D_ALPHA', 'f4'), 

352 ('D_SECZENITH', 'f4'), 

353 ('D_LNPWV_I1', 'f4'), 

354 ('D_O3_I1', 'f4'), 

355 ('D_LNTAU_I1', 'f4'), 

356 ('D_ALPHA_I1', 'f4'), 

357 ('D_SECZENITH_I1', 'f4')]) 

358 

359 for name in lutDerivFlat.dtype.names: 

360 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:] 

361 

362 # The fgcm.FgcmLUT() class copies all the LUT information into special 

363 # shared memory objects that will not blow up the memory usage when used 

364 # with python multiprocessing. Once all the numbers are copied, the 

365 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat) 

366 # will fall out of scope and can be cleaned up by the garbage collector. 

367 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd, 

368 filterToBand=physicalFilterMap) 

369 

370 return fgcmLut, lutIndexVals, lutStd 

371 

372 

373def translateVisitCatalog(visitCat): 

374 """ 

375 Translate the FGCM visit catalog to an fgcm-compatible object 

376 

377 Parameters 

378 ---------- 

379 visitCat: `lsst.afw.table.BaseCatalog` 

380 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask` 

381 

382 Returns 

383 ------- 

384 fgcmExpInfo: `numpy.ndarray` 

385 Numpy array for visit information for FGCM 

386 

387 Notes 

388 ----- 

389 After running this code, it is wise to `del visitCat` to clear the memory. 

390 """ 

391 

392 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'), 

393 ('MJD', 'f8'), 

394 ('EXPTIME', 'f8'), 

395 ('PSFSIGMA', 'f8'), 

396 ('DELTA_APER', 'f8'), 

397 ('SKYBACKGROUND', 'f8'), 

398 ('DEEPFLAG', 'i2'), 

399 ('TELHA', 'f8'), 

400 ('TELRA', 'f8'), 

401 ('TELDEC', 'f8'), 

402 ('TELROT', 'f8'), 

403 ('PMB', 'f8'), 

404 ('FILTERNAME', 'a50')]) 

405 fgcmExpInfo['VISIT'][:] = visitCat['visit'] 

406 fgcmExpInfo['MJD'][:] = visitCat['mjd'] 

407 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime'] 

408 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag'] 

409 fgcmExpInfo['TELHA'][:] = visitCat['telha'] 

410 fgcmExpInfo['TELRA'][:] = visitCat['telra'] 

411 fgcmExpInfo['TELDEC'][:] = visitCat['teldec'] 

412 fgcmExpInfo['TELROT'][:] = visitCat['telrot'] 

413 fgcmExpInfo['PMB'][:] = visitCat['pmb'] 

414 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma'] 

415 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper'] 

416 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground'] 

417 # Note that we have to go through asAstropy() to get a string 

418 # array out of an afwTable. This is faster than a row-by-row loop. 

419 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter'] 

420 

421 return fgcmExpInfo 

422 

423 

424@deprecated(reason="This method is no longer used in fgcmcal. It will be removed after v23.", 

425 version="v23.0", category=FutureWarning) 

426def computeCcdOffsets(camera, defaultOrientation): 

427 """ 

428 Compute the CCD offsets in ra/dec and x/y space 

429 

430 Parameters 

431 ---------- 

432 camera: `lsst.afw.cameraGeom.Camera` 

433 defaultOrientation: `float` 

434 Default camera orientation (degrees) 

435 

436 Returns 

437 ------- 

438 ccdOffsets: `numpy.ndarray` 

439 Numpy array with ccd offset information for input to FGCM. 

440 Angular units are degrees, and x/y units are pixels. 

441 """ 

442 # TODO: DM-21215 will fully generalize to arbitrary camera orientations 

443 

444 # and we need to know the ccd offsets from the camera geometry 

445 ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'), 

446 ('DELTA_RA', 'f8'), 

447 ('DELTA_DEC', 'f8'), 

448 ('RA_SIZE', 'f8'), 

449 ('DEC_SIZE', 'f8'), 

450 ('X_SIZE', 'i4'), 

451 ('Y_SIZE', 'i4')]) 

452 

453 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem, 

454 # since we are looking for relative positions 

455 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

456 

457 # TODO: DM-17597 will update testdata_jointcal so that the test data 

458 # does not have nan as the boresight angle for HSC data. For the 

459 # time being, there is this ungainly hack. 

460 if camera.getName() == 'HSC' and np.isnan(defaultOrientation): 

461 orientation = 270*geom.degrees 

462 else: 

463 orientation = defaultOrientation*geom.degrees 

464 flipX = False 

465 

466 # Create a temporary visitInfo for input to createInitialSkyWcs 

467 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

468 boresightRotAngle=orientation, 

469 rotType=afwImage.RotType.SKY) 

470 

471 for i, detector in enumerate(camera): 

472 ccdOffsets['CCDNUM'][i] = detector.getId() 

473 

474 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

475 

476 detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS)) 

477 ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees() 

478 ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees() 

479 

480 bbox = detector.getBBox() 

481 

482 detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin())) 

483 detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax())) 

484 

485 ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees()) 

486 ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees()) 

487 

488 ccdOffsets['X_SIZE'][i] = bbox.getMaxX() 

489 ccdOffsets['Y_SIZE'][i] = bbox.getMaxY() 

490 

491 return ccdOffsets 

492 

493 

494def computeReferencePixelScale(camera): 

495 """ 

496 Compute the median pixel scale in the camera 

497 

498 Returns 

499 ------- 

500 pixelScale: `float` 

501 Average pixel scale (arcsecond) over the camera 

502 """ 

503 

504 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

505 orientation = 0.0*geom.degrees 

506 flipX = False 

507 

508 # Create a temporary visitInfo for input to createInitialSkyWcs 

509 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

510 boresightRotAngle=orientation, 

511 rotType=afwImage.RotType.SKY) 

512 

513 pixelScales = np.zeros(len(camera)) 

514 for i, detector in enumerate(camera): 

515 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

516 pixelScales[i] = wcs.getPixelScale().asArcseconds() 

517 

518 ok, = np.where(pixelScales > 0.0) 

519 return np.median(pixelScales[ok]) 

520 

521 

522def computeApproxPixelAreaFields(camera): 

523 """ 

524 Compute the approximate pixel area bounded fields from the camera 

525 geometry. 

526 

527 Parameters 

528 ---------- 

529 camera: `lsst.afw.cameraGeom.Camera` 

530 

531 Returns 

532 ------- 

533 approxPixelAreaFields: `dict` 

534 Dictionary of approximate area fields, keyed with detector ID 

535 """ 

536 

537 areaScaling = 1. / computeReferencePixelScale(camera)**2. 

538 

539 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem, 

540 # since we are looking for relative scales 

541 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

542 

543 flipX = False 

544 # Create a temporary visitInfo for input to createInitialSkyWcs 

545 # The orientation does not matter for the area computation 

546 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

547 boresightRotAngle=0.0*geom.degrees, 

548 rotType=afwImage.RotType.SKY) 

549 

550 approxPixelAreaFields = {} 

551 

552 for i, detector in enumerate(camera): 

553 key = detector.getId() 

554 

555 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

556 bbox = detector.getBBox() 

557 

558 areaField = afwMath.PixelAreaBoundedField(bbox, wcs, 

559 unit=geom.arcseconds, scaling=areaScaling) 

560 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField) 

561 

562 approxPixelAreaFields[key] = approxAreaField 

563 

564 return approxPixelAreaFields 

565 

566 

567def makeZptSchema(superStarChebyshevSize, zptChebyshevSize): 

568 """ 

569 Make the zeropoint schema 

570 

571 Parameters 

572 ---------- 

573 superStarChebyshevSize: `int` 

574 Length of the superstar chebyshev array 

575 zptChebyshevSize: `int` 

576 Length of the zeropoint chebyshev array 

577 

578 Returns 

579 ------- 

580 zptSchema: `lsst.afw.table.schema` 

581 """ 

582 

583 zptSchema = afwTable.Schema() 

584 

585 zptSchema.addField('visit', type=np.int64, doc='Visit number') 

586 zptSchema.addField('detector', type=np.int32, doc='Detector ID number') 

587 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: ' 

588 '1: Photometric, used in fit; ' 

589 '2: Photometric, not used in fit; ' 

590 '4: Non-photometric, on partly photometric night; ' 

591 '8: Non-photometric, on non-photometric night; ' 

592 '16: No zeropoint could be determined; ' 

593 '32: Too few stars for reliable gray computation')) 

594 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)') 

595 zptSchema.addField('fgcmZptErr', type=np.float64, 

596 doc='Error on zeropoint, estimated from repeatability + number of obs') 

597 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2, 

598 doc='maximum x/maximum y to scale to apply chebyshev parameters') 

599 zptSchema.addField('fgcmfZptCheb', type='ArrayD', 

600 size=zptChebyshevSize, 

601 doc='Chebyshev parameters (flattened) for zeropoint') 

602 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD', 

603 size=superStarChebyshevSize, 

604 doc='Chebyshev parameters (flattened) for superStarFlat') 

605 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband') 

606 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral') 

607 zptSchema.addField('fgcmR0', type=np.float64, 

608 doc='Retrieved i0 integral, estimated from stars (only for flag 1)') 

609 zptSchema.addField('fgcmR10', type=np.float64, 

610 doc='Retrieved i10 integral, estimated from stars (only for flag 1)') 

611 zptSchema.addField('fgcmGry', type=np.float64, 

612 doc='Estimated gray extinction relative to atmospheric solution; ' 

613 'only for fgcmFlag <= 4 (see fgcmFlag) ') 

614 zptSchema.addField('fgcmDeltaChrom', type=np.float64, 

615 doc='Mean chromatic correction for stars in this ccd; ' 

616 'only for fgcmFlag <= 4 (see fgcmFlag)') 

617 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd') 

618 zptSchema.addField('fgcmTilings', type=np.float64, 

619 doc='Number of photometric tilings used for solution for ccd') 

620 zptSchema.addField('fgcmFpGry', type=np.float64, 

621 doc='Average gray extinction over the full focal plane ' 

622 '(same for all ccds in a visit)') 

623 zptSchema.addField('fgcmFpGryBlue', type=np.float64, 

624 doc='Average gray extinction over the full focal plane ' 

625 'for 25% bluest stars') 

626 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64, 

627 doc='Error on Average gray extinction over the full focal plane ' 

628 'for 25% bluest stars') 

629 zptSchema.addField('fgcmFpGryRed', type=np.float64, 

630 doc='Average gray extinction over the full focal plane ' 

631 'for 25% reddest stars') 

632 zptSchema.addField('fgcmFpGryRedErr', type=np.float64, 

633 doc='Error on Average gray extinction over the full focal plane ' 

634 'for 25% reddest stars') 

635 zptSchema.addField('fgcmFpVar', type=np.float64, 

636 doc='Variance of gray extinction over the full focal plane ' 

637 '(same for all ccds in a visit)') 

638 zptSchema.addField('fgcmDust', type=np.float64, 

639 doc='Gray dust extinction from the primary/corrector' 

640 'at the time of the exposure') 

641 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction') 

642 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm') 

643 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64, 

644 doc=('Local background correction from brightest percentile ' 

645 '(value set by deltaMagBkgOffsetPercentile) calibration ' 

646 'stars.')) 

647 zptSchema.addField('exptime', type=np.float32, doc='Exposure time') 

648 zptSchema.addField('filtername', type=str, size=10, doc='Filter name') 

649 

650 return zptSchema 

651 

652 

653def makeZptCat(zptSchema, zpStruct): 

654 """ 

655 Make the zeropoint catalog for persistence 

656 

657 Parameters 

658 ---------- 

659 zptSchema: `lsst.afw.table.Schema` 

660 Zeropoint catalog schema 

661 zpStruct: `numpy.ndarray` 

662 Zeropoint structure from fgcm 

663 

664 Returns 

665 ------- 

666 zptCat: `afwTable.BaseCatalog` 

667 Zeropoint catalog for persistence 

668 """ 

669 

670 zptCat = afwTable.BaseCatalog(zptSchema) 

671 zptCat.reserve(zpStruct.size) 

672 

673 for filterName in zpStruct['FILTERNAME']: 

674 rec = zptCat.addNew() 

675 rec['filtername'] = filterName.decode('utf-8') 

676 

677 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD] 

678 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD] 

679 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG'] 

680 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT'] 

681 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR'] 

682 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX'] 

683 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB'] 

684 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB'] 

685 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0'] 

686 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10'] 

687 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0'] 

688 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10'] 

689 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY'] 

690 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM'] 

691 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR'] 

692 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS'] 

693 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY'] 

694 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0] 

695 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0] 

696 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2] 

697 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2] 

698 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR'] 

699 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST'] 

700 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT'] 

701 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR'] 

702 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG'] 

703 zptCat['exptime'][:] = zpStruct['EXPTIME'] 

704 

705 return zptCat 

706 

707 

708def makeAtmSchema(): 

709 """ 

710 Make the atmosphere schema 

711 

712 Returns 

713 ------- 

714 atmSchema: `lsst.afw.table.Schema` 

715 """ 

716 

717 atmSchema = afwTable.Schema() 

718 

719 atmSchema.addField('visit', type=np.int32, doc='Visit number') 

720 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)') 

721 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)') 

722 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth') 

723 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope') 

724 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)') 

725 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)') 

726 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor') 

727 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction') 

728 

729 return atmSchema 

730 

731 

732def makeAtmCat(atmSchema, atmStruct): 

733 """ 

734 Make the atmosphere catalog for persistence 

735 

736 Parameters 

737 ---------- 

738 atmSchema: `lsst.afw.table.Schema` 

739 Atmosphere catalog schema 

740 atmStruct: `numpy.ndarray` 

741 Atmosphere structure from fgcm 

742 

743 Returns 

744 ------- 

745 atmCat: `lsst.afw.table.BaseCatalog` 

746 Atmosphere catalog for persistence 

747 """ 

748 

749 atmCat = afwTable.BaseCatalog(atmSchema) 

750 atmCat.resize(atmStruct.size) 

751 

752 atmCat['visit'][:] = atmStruct['VISIT'] 

753 atmCat['pmb'][:] = atmStruct['PMB'] 

754 atmCat['pwv'][:] = atmStruct['PWV'] 

755 atmCat['tau'][:] = atmStruct['TAU'] 

756 atmCat['alpha'][:] = atmStruct['ALPHA'] 

757 atmCat['o3'][:] = atmStruct['O3'] 

758 atmCat['secZenith'][:] = atmStruct['SECZENITH'] 

759 atmCat['cTrans'][:] = atmStruct['CTRANS'] 

760 atmCat['lamStd'][:] = atmStruct['LAMSTD'] 

761 

762 return atmCat 

763 

764 

765def makeStdSchema(nBands): 

766 """ 

767 Make the standard star schema 

768 

769 Parameters 

770 ---------- 

771 nBands: `int` 

772 Number of bands in standard star catalog 

773 

774 Returns 

775 ------- 

776 stdSchema: `lsst.afw.table.Schema` 

777 """ 

778 

779 stdSchema = afwTable.SimpleTable.makeMinimalSchema() 

780 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations', 

781 size=nBands) 

782 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations', 

783 size=nBands) 

784 stdSchema.addField('mag_std_noabs', type='ArrayF', 

785 doc='Standard magnitude (no absolute calibration)', 

786 size=nBands) 

787 stdSchema.addField('magErr_std', type='ArrayF', 

788 doc='Standard magnitude error', 

789 size=nBands) 

790 stdSchema.addField('npsfcand', type='ArrayI', 

791 doc='Number of observations flagged as psf candidates', 

792 size=nBands) 

793 stdSchema.addField('delta_aper', type='ArrayF', 

794 doc='Delta mag (small - large aperture)', 

795 size=nBands) 

796 

797 return stdSchema 

798 

799 

800def makeStdCat(stdSchema, stdStruct, goodBands): 

801 """ 

802 Make the standard star catalog for persistence 

803 

804 Parameters 

805 ---------- 

806 stdSchema: `lsst.afw.table.Schema` 

807 Standard star catalog schema 

808 stdStruct: `numpy.ndarray` 

809 Standard star structure in FGCM format 

810 goodBands: `list` 

811 List of good band names used in stdStruct 

812 

813 Returns 

814 ------- 

815 stdCat: `lsst.afw.table.BaseCatalog` 

816 Standard star catalog for persistence 

817 """ 

818 

819 stdCat = afwTable.SimpleCatalog(stdSchema) 

820 stdCat.resize(stdStruct.size) 

821 

822 stdCat['id'][:] = stdStruct['FGCM_ID'] 

823 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees 

824 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees 

825 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :] 

826 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :] 

827 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :] 

828 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :] 

829 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :] 

830 stdCat['delta_aper'][:, :] = stdStruct['DELTA_APER'][:, :] 

831 

832 md = PropertyList() 

833 md.set("BANDS", list(goodBands)) 

834 stdCat.setMetadata(md) 

835 

836 return stdCat 

837 

838 

839def computeApertureRadiusFromName(fluxField): 

840 """ 

841 Compute the radius associated with a CircularApertureFlux or ApFlux field. 

842 

843 Parameters 

844 ---------- 

845 fluxField : `str` 

846 CircularApertureFlux or ApFlux 

847 

848 Returns 

849 ------- 

850 apertureRadius : `float` 

851 Radius of the aperture field, in pixels. 

852 

853 Raises 

854 ------ 

855 RuntimeError: Raised if flux field is not a CircularApertureFlux, 

856 ApFlux, or apFlux. 

857 """ 

858 # TODO: Move this method to more general stack method in DM-25775 

859 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField) 

860 

861 if m is None: 

862 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux") 

863 

864 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10. 

865 

866 return apertureRadius 

867 

868 

869def extractReferenceMags(refStars, bands, filterMap): 

870 """ 

871 Extract reference magnitudes from refStars for given bands and 

872 associated filterMap. 

873 

874 Parameters 

875 ---------- 

876 refStars : `lsst.afw.table.BaseCatalog` 

877 FGCM reference star catalog 

878 bands : `list` 

879 List of bands for calibration 

880 filterMap: `dict` 

881 FGCM mapping of filter to band 

882 

883 Returns 

884 ------- 

885 refMag : `np.ndarray` 

886 nstar x nband array of reference magnitudes 

887 refMagErr : `np.ndarray` 

888 nstar x nband array of reference magnitude errors 

889 """ 

890 # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent 

891 # against index errors and allow more flexibility in fitting after 

892 # the build stars step. 

893 

894 md = refStars.getMetadata() 

895 if 'FILTERNAMES' in md: 

896 filternames = md.getArray('FILTERNAMES') 

897 

898 # The reference catalog that fgcm wants has one entry per band 

899 # in the config file 

900 refMag = np.zeros((len(refStars), len(bands)), 

901 dtype=refStars['refMag'].dtype) + 99.0 

902 refMagErr = np.zeros_like(refMag) + 99.0 

903 for i, filtername in enumerate(filternames): 

904 # We are allowed to run the fit configured so that we do not 

905 # use every column in the reference catalog. 

906 try: 

907 band = filterMap[filtername] 

908 except KeyError: 

909 continue 

910 try: 

911 ind = bands.index(band) 

912 except ValueError: 

913 continue 

914 

915 refMag[:, ind] = refStars['refMag'][:, i] 

916 refMagErr[:, ind] = refStars['refMagErr'][:, i] 

917 

918 else: 

919 # Continue to use old catalogs as before. 

920 refMag = refStars['refMag'][:, :] 

921 refMagErr = refStars['refMagErr'][:, :] 

922 

923 return refMag, refMagErr 

924 

925 

926def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

927 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

928 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

929 

930 return registry.queryDatasets(datasetType, 

931 dataId=quantumDataId, 

932 collections=[unboundedCollection])