Coverage for python/lsst/fgcmcal/utilities.py: 10%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

294 statements  

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Utility functions for fgcmcal. 

22 

23This file contains utility functions that are used by more than one task, 

24and do not need to be part of a task. 

25""" 

26 

27import numpy as np 

28import os 

29import re 

30 

31from lsst.daf.base import PropertyList 

32import lsst.daf.persistence as dafPersist 

33import lsst.afw.cameraGeom as afwCameraGeom 

34import lsst.afw.table as afwTable 

35import lsst.afw.image as afwImage 

36import lsst.afw.math as afwMath 

37import lsst.geom as geom 

38from lsst.obs.base import createInitialSkyWcs 

39from lsst.obs.base import Instrument 

40 

41import fgcm 

42 

43 

44FGCM_EXP_FIELD = 'VISIT' 

45FGCM_CCD_FIELD = 'DETECTOR' 

46FGCM_ILLEGAL_VALUE = -9999.0 

47 

48 

49def makeConfigDict(config, log, camera, maxIter, 

50 resetFitParameters, outputZeropoints, 

51 lutFilterNames, tract=None): 

52 """ 

53 Make the FGCM fit cycle configuration dict 

54 

55 Parameters 

56 ---------- 

57 config: `lsst.fgcmcal.FgcmFitCycleConfig` 

58 Configuration object 

59 log: `lsst.log.Log` 

60 LSST log object 

61 camera: `lsst.afw.cameraGeom.Camera` 

62 Camera from the butler 

63 maxIter: `int` 

64 Maximum number of iterations 

65 resetFitParameters: `bool` 

66 Reset fit parameters before fitting? 

67 outputZeropoints: `bool` 

68 Compute zeropoints for output? 

69 lutFilterNames : array-like, `str` 

70 Array of physical filter names in the LUT. 

71 tract: `int`, optional 

72 Tract number for extending the output file name for debugging. 

73 Default is None. 

74 

75 Returns 

76 ------- 

77 configDict: `dict` 

78 Configuration dictionary for fgcm 

79 """ 

80 # Extract the bands that are _not_ being fit for fgcm configuration 

81 notFitBands = [b for b in config.bands if b not in config.fitBands] 

82 

83 # process the starColorCuts 

84 starColorCutList = [] 

85 for ccut in config.starColorCuts: 

86 parts = ccut.split(',') 

87 starColorCutList.append([parts[0], parts[1], float(parts[2]), float(parts[3])]) 

88 

89 # TODO: Having direct access to the mirror area from the camera would be 

90 # useful. See DM-16489. 

91 # Mirror area in cm**2 

92 mirrorArea = np.pi*(camera.telescopeDiameter*100./2.)**2. 

93 

94 # Get approximate average camera gain: 

95 gains = [amp.getGain() for detector in camera for amp in detector.getAmplifiers()] 

96 cameraGain = float(np.median(gains)) 

97 

98 # Cut down the filter map to those that are in the LUT 

99 filterToBand = {filterName: config.physicalFilterMap[filterName] for 

100 filterName in lutFilterNames} 

101 

102 if tract is None: 

103 outfileBase = config.outfileBase 

104 else: 

105 outfileBase = '%s-%06d' % (config.outfileBase, tract) 

106 

107 # create a configuration dictionary for fgcmFitCycle 

108 configDict = {'outfileBase': outfileBase, 

109 'logger': log, 

110 'exposureFile': None, 

111 'obsFile': None, 

112 'indexFile': None, 

113 'lutFile': None, 

114 'mirrorArea': mirrorArea, 

115 'cameraGain': cameraGain, 

116 'ccdStartIndex': camera[0].getId(), 

117 'expField': FGCM_EXP_FIELD, 

118 'ccdField': FGCM_CCD_FIELD, 

119 'seeingField': 'DELTA_APER', 

120 'fwhmField': 'PSFSIGMA', 

121 'skyBrightnessField': 'SKYBACKGROUND', 

122 'deepFlag': 'DEEPFLAG', # unused 

123 'bands': list(config.bands), 

124 'fitBands': list(config.fitBands), 

125 'notFitBands': notFitBands, 

126 'requiredBands': list(config.requiredBands), 

127 'filterToBand': filterToBand, 

128 'logLevel': 'INFO', 

129 'nCore': config.nCore, 

130 'nStarPerRun': config.nStarPerRun, 

131 'nExpPerRun': config.nExpPerRun, 

132 'reserveFraction': config.reserveFraction, 

133 'freezeStdAtmosphere': config.freezeStdAtmosphere, 

134 'precomputeSuperStarInitialCycle': config.precomputeSuperStarInitialCycle, 

135 'superStarSubCCDDict': dict(config.superStarSubCcdDict), 

136 'superStarSubCCDChebyshevOrder': config.superStarSubCcdChebyshevOrder, 

137 'superStarSubCCDTriangular': config.superStarSubCcdTriangular, 

138 'superStarSigmaClip': config.superStarSigmaClip, 

139 'focalPlaneSigmaClip': config.focalPlaneSigmaClip, 

140 'ccdGraySubCCDDict': dict(config.ccdGraySubCcdDict), 

141 'ccdGraySubCCDChebyshevOrder': config.ccdGraySubCcdChebyshevOrder, 

142 'ccdGraySubCCDTriangular': config.ccdGraySubCcdTriangular, 

143 'ccdGrayFocalPlaneDict': dict(config.ccdGrayFocalPlaneDict), 

144 'ccdGrayFocalPlaneChebyshevOrder': config.ccdGrayFocalPlaneChebyshevOrder, 

145 'ccdGrayFocalPlaneFitMinCcd': config.ccdGrayFocalPlaneFitMinCcd, 

146 'cycleNumber': config.cycleNumber, 

147 'maxIter': maxIter, 

148 'deltaMagBkgOffsetPercentile': config.deltaMagBkgOffsetPercentile, 

149 'deltaMagBkgPerCcd': config.deltaMagBkgPerCcd, 

150 'UTBoundary': config.utBoundary, 

151 'washMJDs': config.washMjds, 

152 'epochMJDs': config.epochMjds, 

153 'coatingMJDs': config.coatingMjds, 

154 'minObsPerBand': config.minObsPerBand, 

155 'latitude': config.latitude, 

156 'brightObsGrayMax': config.brightObsGrayMax, 

157 'minStarPerCCD': config.minStarPerCcd, 

158 'minCCDPerExp': config.minCcdPerExp, 

159 'maxCCDGrayErr': config.maxCcdGrayErr, 

160 'minStarPerExp': config.minStarPerExp, 

161 'minExpPerNight': config.minExpPerNight, 

162 'expGrayInitialCut': config.expGrayInitialCut, 

163 'expGrayPhotometricCutDict': dict(config.expGrayPhotometricCutDict), 

164 'expGrayHighCutDict': dict(config.expGrayHighCutDict), 

165 'expGrayRecoverCut': config.expGrayRecoverCut, 

166 'expVarGrayPhotometricCutDict': dict(config.expVarGrayPhotometricCutDict), 

167 'expGrayErrRecoverCut': config.expGrayErrRecoverCut, 

168 'refStarSnMin': config.refStarSnMin, 

169 'refStarOutlierNSig': config.refStarOutlierNSig, 

170 'applyRefStarColorCuts': config.applyRefStarColorCuts, 

171 'illegalValue': FGCM_ILLEGAL_VALUE, # internally used by fgcm. 

172 'starColorCuts': starColorCutList, 

173 'aperCorrFitNBins': config.aperCorrFitNBins, 

174 'aperCorrInputSlopeDict': dict(config.aperCorrInputSlopeDict), 

175 'sedBoundaryTermDict': config.sedboundaryterms.toDict()['data'], 

176 'sedTermDict': config.sedterms.toDict()['data'], 

177 'colorSplitBands': list(config.colorSplitBands), 

178 'sigFgcmMaxErr': config.sigFgcmMaxErr, 

179 'sigFgcmMaxEGrayDict': dict(config.sigFgcmMaxEGrayDict), 

180 'ccdGrayMaxStarErr': config.ccdGrayMaxStarErr, 

181 'approxThroughputDict': dict(config.approxThroughputDict), 

182 'sigmaCalRange': list(config.sigmaCalRange), 

183 'sigmaCalFitPercentile': list(config.sigmaCalFitPercentile), 

184 'sigmaCalPlotPercentile': list(config.sigmaCalPlotPercentile), 

185 'sigma0Phot': config.sigma0Phot, 

186 'mapLongitudeRef': config.mapLongitudeRef, 

187 'mapNSide': config.mapNSide, 

188 'varNSig': 100.0, # Turn off 'variable star selection' which doesn't work yet 

189 'varMinBand': 2, 

190 'useRetrievedPwv': False, 

191 'useNightlyRetrievedPwv': False, 

192 'pwvRetrievalSmoothBlock': 25, 

193 'useQuadraticPwv': config.useQuadraticPwv, 

194 'useRetrievedTauInit': False, 

195 'tauRetrievalMinCCDPerNight': 500, 

196 'modelMagErrors': config.modelMagErrors, 

197 'instrumentParsPerBand': config.instrumentParsPerBand, 

198 'instrumentSlopeMinDeltaT': config.instrumentSlopeMinDeltaT, 

199 'fitMirrorChromaticity': config.fitMirrorChromaticity, 

200 'useRepeatabilityForExpGrayCutsDict': dict(config.useRepeatabilityForExpGrayCutsDict), 

201 'autoPhotometricCutNSig': config.autoPhotometricCutNSig, 

202 'autoHighCutNSig': config.autoHighCutNSig, 

203 'printOnly': False, 

204 'quietMode': config.quietMode, 

205 'randomSeed': config.randomSeed, 

206 'outputStars': False, 

207 'outputPath': os.path.abspath('.'), 

208 'clobber': True, 

209 'useSedLUT': False, 

210 'resetParameters': resetFitParameters, 

211 'doPlots': config.doPlots, 

212 'outputFgcmcalZpts': True, # when outputting zpts, use fgcmcal format 

213 'outputZeropoints': outputZeropoints} 

214 

215 return configDict 

216 

217 

218def translateFgcmLut(lutCat, physicalFilterMap): 

219 """ 

220 Translate the FGCM look-up-table into an fgcm-compatible object 

221 

222 Parameters 

223 ---------- 

224 lutCat: `lsst.afw.table.BaseCatalog` 

225 Catalog describing the FGCM look-up table 

226 physicalFilterMap: `dict` 

227 Physical filter to band mapping 

228 

229 Returns 

230 ------- 

231 fgcmLut: `lsst.fgcm.FgcmLut` 

232 Lookup table for FGCM 

233 lutIndexVals: `numpy.ndarray` 

234 Numpy array with LUT index information for FGCM 

235 lutStd: `numpy.ndarray` 

236 Numpy array with LUT standard throughput values for FGCM 

237 

238 Notes 

239 ----- 

240 After running this code, it is wise to `del lutCat` to clear the memory. 

241 """ 

242 

243 # first we need the lutIndexVals 

244 lutFilterNames = np.array(lutCat[0]['physicalFilters'].split(','), dtype='U') 

245 lutStdFilterNames = np.array(lutCat[0]['stdPhysicalFilters'].split(','), dtype='U') 

246 

247 # Note that any discrepancies between config values will raise relevant 

248 # exceptions in the FGCM code. 

249 

250 lutIndexVals = np.zeros(1, dtype=[('FILTERNAMES', lutFilterNames.dtype.str, 

251 lutFilterNames.size), 

252 ('STDFILTERNAMES', lutStdFilterNames.dtype.str, 

253 lutStdFilterNames.size), 

254 ('PMB', 'f8', lutCat[0]['pmb'].size), 

255 ('PMBFACTOR', 'f8', lutCat[0]['pmbFactor'].size), 

256 ('PMBELEVATION', 'f8'), 

257 ('LAMBDANORM', 'f8'), 

258 ('PWV', 'f8', lutCat[0]['pwv'].size), 

259 ('O3', 'f8', lutCat[0]['o3'].size), 

260 ('TAU', 'f8', lutCat[0]['tau'].size), 

261 ('ALPHA', 'f8', lutCat[0]['alpha'].size), 

262 ('ZENITH', 'f8', lutCat[0]['zenith'].size), 

263 ('NCCD', 'i4')]) 

264 

265 lutIndexVals['FILTERNAMES'][:] = lutFilterNames 

266 lutIndexVals['STDFILTERNAMES'][:] = lutStdFilterNames 

267 lutIndexVals['PMB'][:] = lutCat[0]['pmb'] 

268 lutIndexVals['PMBFACTOR'][:] = lutCat[0]['pmbFactor'] 

269 lutIndexVals['PMBELEVATION'] = lutCat[0]['pmbElevation'] 

270 lutIndexVals['LAMBDANORM'] = lutCat[0]['lambdaNorm'] 

271 lutIndexVals['PWV'][:] = lutCat[0]['pwv'] 

272 lutIndexVals['O3'][:] = lutCat[0]['o3'] 

273 lutIndexVals['TAU'][:] = lutCat[0]['tau'] 

274 lutIndexVals['ALPHA'][:] = lutCat[0]['alpha'] 

275 lutIndexVals['ZENITH'][:] = lutCat[0]['zenith'] 

276 lutIndexVals['NCCD'] = lutCat[0]['nCcd'] 

277 

278 # now we need the Standard Values 

279 lutStd = np.zeros(1, dtype=[('PMBSTD', 'f8'), 

280 ('PWVSTD', 'f8'), 

281 ('O3STD', 'f8'), 

282 ('TAUSTD', 'f8'), 

283 ('ALPHASTD', 'f8'), 

284 ('ZENITHSTD', 'f8'), 

285 ('LAMBDARANGE', 'f8', 2), 

286 ('LAMBDASTEP', 'f8'), 

287 ('LAMBDASTD', 'f8', lutFilterNames.size), 

288 ('LAMBDASTDFILTER', 'f8', lutStdFilterNames.size), 

289 ('I0STD', 'f8', lutFilterNames.size), 

290 ('I1STD', 'f8', lutFilterNames.size), 

291 ('I10STD', 'f8', lutFilterNames.size), 

292 ('I2STD', 'f8', lutFilterNames.size), 

293 ('LAMBDAB', 'f8', lutFilterNames.size), 

294 ('ATMLAMBDA', 'f8', lutCat[0]['atmLambda'].size), 

295 ('ATMSTDTRANS', 'f8', lutCat[0]['atmStdTrans'].size)]) 

296 lutStd['PMBSTD'] = lutCat[0]['pmbStd'] 

297 lutStd['PWVSTD'] = lutCat[0]['pwvStd'] 

298 lutStd['O3STD'] = lutCat[0]['o3Std'] 

299 lutStd['TAUSTD'] = lutCat[0]['tauStd'] 

300 lutStd['ALPHASTD'] = lutCat[0]['alphaStd'] 

301 lutStd['ZENITHSTD'] = lutCat[0]['zenithStd'] 

302 lutStd['LAMBDARANGE'][:] = lutCat[0]['lambdaRange'][:] 

303 lutStd['LAMBDASTEP'] = lutCat[0]['lambdaStep'] 

304 lutStd['LAMBDASTD'][:] = lutCat[0]['lambdaStd'] 

305 lutStd['LAMBDASTDFILTER'][:] = lutCat[0]['lambdaStdFilter'] 

306 lutStd['I0STD'][:] = lutCat[0]['i0Std'] 

307 lutStd['I1STD'][:] = lutCat[0]['i1Std'] 

308 lutStd['I10STD'][:] = lutCat[0]['i10Std'] 

309 lutStd['I2STD'][:] = lutCat[0]['i2Std'] 

310 lutStd['LAMBDAB'][:] = lutCat[0]['lambdaB'] 

311 lutStd['ATMLAMBDA'][:] = lutCat[0]['atmLambda'][:] 

312 lutStd['ATMSTDTRANS'][:] = lutCat[0]['atmStdTrans'][:] 

313 

314 lutTypes = [row['luttype'] for row in lutCat] 

315 

316 # And the flattened look-up-table 

317 lutFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('I0', 'f4'), 

318 ('I1', 'f4')]) 

319 

320 lutFlat['I0'][:] = lutCat[lutTypes.index('I0')]['lut'][:] 

321 lutFlat['I1'][:] = lutCat[lutTypes.index('I1')]['lut'][:] 

322 

323 lutDerivFlat = np.zeros(lutCat[0]['lut'].size, dtype=[('D_LNPWV', 'f4'), 

324 ('D_O3', 'f4'), 

325 ('D_LNTAU', 'f4'), 

326 ('D_ALPHA', 'f4'), 

327 ('D_SECZENITH', 'f4'), 

328 ('D_LNPWV_I1', 'f4'), 

329 ('D_O3_I1', 'f4'), 

330 ('D_LNTAU_I1', 'f4'), 

331 ('D_ALPHA_I1', 'f4'), 

332 ('D_SECZENITH_I1', 'f4')]) 

333 

334 for name in lutDerivFlat.dtype.names: 

335 lutDerivFlat[name][:] = lutCat[lutTypes.index(name)]['lut'][:] 

336 

337 # The fgcm.FgcmLUT() class copies all the LUT information into special 

338 # shared memory objects that will not blow up the memory usage when used 

339 # with python multiprocessing. Once all the numbers are copied, the 

340 # references to the temporary objects (lutCat, lutFlat, lutDerivFlat) 

341 # will fall out of scope and can be cleaned up by the garbage collector. 

342 fgcmLut = fgcm.FgcmLUT(lutIndexVals, lutFlat, lutDerivFlat, lutStd, 

343 filterToBand=physicalFilterMap) 

344 

345 return fgcmLut, lutIndexVals, lutStd 

346 

347 

348def translateVisitCatalog(visitCat): 

349 """ 

350 Translate the FGCM visit catalog to an fgcm-compatible object 

351 

352 Parameters 

353 ---------- 

354 visitCat: `lsst.afw.table.BaseCatalog` 

355 FGCM visitCat from `lsst.fgcmcal.FgcmBuildStarsTask` 

356 

357 Returns 

358 ------- 

359 fgcmExpInfo: `numpy.ndarray` 

360 Numpy array for visit information for FGCM 

361 

362 Notes 

363 ----- 

364 After running this code, it is wise to `del visitCat` to clear the memory. 

365 """ 

366 

367 fgcmExpInfo = np.zeros(len(visitCat), dtype=[('VISIT', 'i8'), 

368 ('MJD', 'f8'), 

369 ('EXPTIME', 'f8'), 

370 ('PSFSIGMA', 'f8'), 

371 ('DELTA_APER', 'f8'), 

372 ('SKYBACKGROUND', 'f8'), 

373 ('DEEPFLAG', 'i2'), 

374 ('TELHA', 'f8'), 

375 ('TELRA', 'f8'), 

376 ('TELDEC', 'f8'), 

377 ('TELROT', 'f8'), 

378 ('PMB', 'f8'), 

379 ('FILTERNAME', 'a50')]) 

380 fgcmExpInfo['VISIT'][:] = visitCat['visit'] 

381 fgcmExpInfo['MJD'][:] = visitCat['mjd'] 

382 fgcmExpInfo['EXPTIME'][:] = visitCat['exptime'] 

383 fgcmExpInfo['DEEPFLAG'][:] = visitCat['deepFlag'] 

384 fgcmExpInfo['TELHA'][:] = visitCat['telha'] 

385 fgcmExpInfo['TELRA'][:] = visitCat['telra'] 

386 fgcmExpInfo['TELDEC'][:] = visitCat['teldec'] 

387 fgcmExpInfo['TELROT'][:] = visitCat['telrot'] 

388 fgcmExpInfo['PMB'][:] = visitCat['pmb'] 

389 fgcmExpInfo['PSFSIGMA'][:] = visitCat['psfSigma'] 

390 fgcmExpInfo['DELTA_APER'][:] = visitCat['deltaAper'] 

391 fgcmExpInfo['SKYBACKGROUND'][:] = visitCat['skyBackground'] 

392 # Note that we have to go through asAstropy() to get a string 

393 # array out of an afwTable. This is faster than a row-by-row loop. 

394 fgcmExpInfo['FILTERNAME'][:] = visitCat.asAstropy()['physicalFilter'] 

395 

396 return fgcmExpInfo 

397 

398 

399def computeCcdOffsets(camera, defaultOrientation): 

400 """ 

401 Compute the CCD offsets in ra/dec and x/y space 

402 

403 Parameters 

404 ---------- 

405 camera: `lsst.afw.cameraGeom.Camera` 

406 defaultOrientation: `float` 

407 Default camera orientation (degrees) 

408 

409 Returns 

410 ------- 

411 ccdOffsets: `numpy.ndarray` 

412 Numpy array with ccd offset information for input to FGCM. 

413 Angular units are degrees, and x/y units are pixels. 

414 """ 

415 # TODO: DM-21215 will fully generalize to arbitrary camera orientations 

416 

417 # and we need to know the ccd offsets from the camera geometry 

418 ccdOffsets = np.zeros(len(camera), dtype=[('CCDNUM', 'i4'), 

419 ('DELTA_RA', 'f8'), 

420 ('DELTA_DEC', 'f8'), 

421 ('RA_SIZE', 'f8'), 

422 ('DEC_SIZE', 'f8'), 

423 ('X_SIZE', 'i4'), 

424 ('Y_SIZE', 'i4')]) 

425 

426 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem, 

427 # since we are looking for relative positions 

428 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

429 

430 # TODO: DM-17597 will update testdata_jointcal so that the test data 

431 # does not have nan as the boresight angle for HSC data. For the 

432 # time being, there is this ungainly hack. 

433 if camera.getName() == 'HSC' and np.isnan(defaultOrientation): 

434 orientation = 270*geom.degrees 

435 else: 

436 orientation = defaultOrientation*geom.degrees 

437 flipX = False 

438 

439 # Create a temporary visitInfo for input to createInitialSkyWcs 

440 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

441 boresightRotAngle=orientation, 

442 rotType=afwImage.RotType.SKY) 

443 

444 for i, detector in enumerate(camera): 

445 ccdOffsets['CCDNUM'][i] = detector.getId() 

446 

447 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

448 

449 detCenter = wcs.pixelToSky(detector.getCenter(afwCameraGeom.PIXELS)) 

450 ccdOffsets['DELTA_RA'][i] = (detCenter.getRa() - boresight.getRa()).asDegrees() 

451 ccdOffsets['DELTA_DEC'][i] = (detCenter.getDec() - boresight.getDec()).asDegrees() 

452 

453 bbox = detector.getBBox() 

454 

455 detCorner1 = wcs.pixelToSky(geom.Point2D(bbox.getMin())) 

456 detCorner2 = wcs.pixelToSky(geom.Point2D(bbox.getMax())) 

457 

458 ccdOffsets['RA_SIZE'][i] = np.abs((detCorner2.getRa() - detCorner1.getRa()).asDegrees()) 

459 ccdOffsets['DEC_SIZE'][i] = np.abs((detCorner2.getDec() - detCorner1.getDec()).asDegrees()) 

460 

461 ccdOffsets['X_SIZE'][i] = bbox.getMaxX() 

462 ccdOffsets['Y_SIZE'][i] = bbox.getMaxY() 

463 

464 return ccdOffsets 

465 

466 

467def computeReferencePixelScale(camera): 

468 """ 

469 Compute the median pixel scale in the camera 

470 

471 Returns 

472 ------- 

473 pixelScale: `float` 

474 Average pixel scale (arcsecond) over the camera 

475 """ 

476 

477 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

478 orientation = 0.0*geom.degrees 

479 flipX = False 

480 

481 # Create a temporary visitInfo for input to createInitialSkyWcs 

482 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

483 boresightRotAngle=orientation, 

484 rotType=afwImage.RotType.SKY) 

485 

486 pixelScales = np.zeros(len(camera)) 

487 for i, detector in enumerate(camera): 

488 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

489 pixelScales[i] = wcs.getPixelScale().asArcseconds() 

490 

491 ok, = np.where(pixelScales > 0.0) 

492 return np.median(pixelScales[ok]) 

493 

494 

495def computeApproxPixelAreaFields(camera): 

496 """ 

497 Compute the approximate pixel area bounded fields from the camera 

498 geometry. 

499 

500 Parameters 

501 ---------- 

502 camera: `lsst.afw.cameraGeom.Camera` 

503 

504 Returns 

505 ------- 

506 approxPixelAreaFields: `dict` 

507 Dictionary of approximate area fields, keyed with detector ID 

508 """ 

509 

510 areaScaling = 1. / computeReferencePixelScale(camera)**2. 

511 

512 # Generate fake WCSs centered at 180/0 to avoid the RA=0/360 problem, 

513 # since we are looking for relative scales 

514 boresight = geom.SpherePoint(180.0*geom.degrees, 0.0*geom.degrees) 

515 

516 flipX = False 

517 # Create a temporary visitInfo for input to createInitialSkyWcs 

518 # The orientation does not matter for the area computation 

519 visitInfo = afwImage.VisitInfo(boresightRaDec=boresight, 

520 boresightRotAngle=0.0*geom.degrees, 

521 rotType=afwImage.RotType.SKY) 

522 

523 approxPixelAreaFields = {} 

524 

525 for i, detector in enumerate(camera): 

526 key = detector.getId() 

527 

528 wcs = createInitialSkyWcs(visitInfo, detector, flipX) 

529 bbox = detector.getBBox() 

530 

531 areaField = afwMath.PixelAreaBoundedField(bbox, wcs, 

532 unit=geom.arcseconds, scaling=areaScaling) 

533 approxAreaField = afwMath.ChebyshevBoundedField.approximate(areaField) 

534 

535 approxPixelAreaFields[key] = approxAreaField 

536 

537 return approxPixelAreaFields 

538 

539 

540def makeZptSchema(superStarChebyshevSize, zptChebyshevSize): 

541 """ 

542 Make the zeropoint schema 

543 

544 Parameters 

545 ---------- 

546 superStarChebyshevSize: `int` 

547 Length of the superstar chebyshev array 

548 zptChebyshevSize: `int` 

549 Length of the zeropoint chebyshev array 

550 

551 Returns 

552 ------- 

553 zptSchema: `lsst.afw.table.schema` 

554 """ 

555 

556 zptSchema = afwTable.Schema() 

557 

558 zptSchema.addField('visit', type=np.int32, doc='Visit number') 

559 zptSchema.addField('detector', type=np.int32, doc='Detector ID number') 

560 zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: ' 

561 '1: Photometric, used in fit; ' 

562 '2: Photometric, not used in fit; ' 

563 '4: Non-photometric, on partly photometric night; ' 

564 '8: Non-photometric, on non-photometric night; ' 

565 '16: No zeropoint could be determined; ' 

566 '32: Too few stars for reliable gray computation')) 

567 zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)') 

568 zptSchema.addField('fgcmZptErr', type=np.float64, 

569 doc='Error on zeropoint, estimated from repeatability + number of obs') 

570 zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2, 

571 doc='maximum x/maximum y to scale to apply chebyshev parameters') 

572 zptSchema.addField('fgcmfZptCheb', type='ArrayD', 

573 size=zptChebyshevSize, 

574 doc='Chebyshev parameters (flattened) for zeropoint') 

575 zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD', 

576 size=superStarChebyshevSize, 

577 doc='Chebyshev parameters (flattened) for superStarFlat') 

578 zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband') 

579 zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral') 

580 zptSchema.addField('fgcmR0', type=np.float64, 

581 doc='Retrieved i0 integral, estimated from stars (only for flag 1)') 

582 zptSchema.addField('fgcmR10', type=np.float64, 

583 doc='Retrieved i10 integral, estimated from stars (only for flag 1)') 

584 zptSchema.addField('fgcmGry', type=np.float64, 

585 doc='Estimated gray extinction relative to atmospheric solution; ' 

586 'only for fgcmFlag <= 4 (see fgcmFlag) ') 

587 zptSchema.addField('fgcmDeltaChrom', type=np.float64, 

588 doc='Mean chromatic correction for stars in this ccd; ' 

589 'only for fgcmFlag <= 4 (see fgcmFlag)') 

590 zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd') 

591 zptSchema.addField('fgcmTilings', type=np.float64, 

592 doc='Number of photometric tilings used for solution for ccd') 

593 zptSchema.addField('fgcmFpGry', type=np.float64, 

594 doc='Average gray extinction over the full focal plane ' 

595 '(same for all ccds in a visit)') 

596 zptSchema.addField('fgcmFpGryBlue', type=np.float64, 

597 doc='Average gray extinction over the full focal plane ' 

598 'for 25% bluest stars') 

599 zptSchema.addField('fgcmFpGryBlueErr', type=np.float64, 

600 doc='Error on Average gray extinction over the full focal plane ' 

601 'for 25% bluest stars') 

602 zptSchema.addField('fgcmFpGryRed', type=np.float64, 

603 doc='Average gray extinction over the full focal plane ' 

604 'for 25% reddest stars') 

605 zptSchema.addField('fgcmFpGryRedErr', type=np.float64, 

606 doc='Error on Average gray extinction over the full focal plane ' 

607 'for 25% reddest stars') 

608 zptSchema.addField('fgcmFpVar', type=np.float64, 

609 doc='Variance of gray extinction over the full focal plane ' 

610 '(same for all ccds in a visit)') 

611 zptSchema.addField('fgcmDust', type=np.float64, 

612 doc='Gray dust extinction from the primary/corrector' 

613 'at the time of the exposure') 

614 zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction') 

615 zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm') 

616 zptSchema.addField('fgcmDeltaMagBkg', type=np.float64, 

617 doc=('Local background correction from brightest percentile ' 

618 '(value set by deltaMagBkgOffsetPercentile) calibration ' 

619 'stars.')) 

620 zptSchema.addField('exptime', type=np.float32, doc='Exposure time') 

621 zptSchema.addField('filtername', type=str, size=10, doc='Filter name') 

622 

623 return zptSchema 

624 

625 

626def makeZptCat(zptSchema, zpStruct): 

627 """ 

628 Make the zeropoint catalog for persistence 

629 

630 Parameters 

631 ---------- 

632 zptSchema: `lsst.afw.table.Schema` 

633 Zeropoint catalog schema 

634 zpStruct: `numpy.ndarray` 

635 Zeropoint structure from fgcm 

636 

637 Returns 

638 ------- 

639 zptCat: `afwTable.BaseCatalog` 

640 Zeropoint catalog for persistence 

641 """ 

642 

643 zptCat = afwTable.BaseCatalog(zptSchema) 

644 zptCat.reserve(zpStruct.size) 

645 

646 for filterName in zpStruct['FILTERNAME']: 

647 rec = zptCat.addNew() 

648 rec['filtername'] = filterName.decode('utf-8') 

649 

650 zptCat['visit'][:] = zpStruct[FGCM_EXP_FIELD] 

651 zptCat['detector'][:] = zpStruct[FGCM_CCD_FIELD] 

652 zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG'] 

653 zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT'] 

654 zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR'] 

655 zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX'] 

656 zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB'] 

657 zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB'] 

658 zptCat['fgcmI0'][:] = zpStruct['FGCM_I0'] 

659 zptCat['fgcmI10'][:] = zpStruct['FGCM_I10'] 

660 zptCat['fgcmR0'][:] = zpStruct['FGCM_R0'] 

661 zptCat['fgcmR10'][:] = zpStruct['FGCM_R10'] 

662 zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY'] 

663 zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM'] 

664 zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR'] 

665 zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS'] 

666 zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY'] 

667 zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0] 

668 zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0] 

669 zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2] 

670 zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2] 

671 zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR'] 

672 zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST'] 

673 zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT'] 

674 zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR'] 

675 zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG'] 

676 zptCat['exptime'][:] = zpStruct['EXPTIME'] 

677 

678 return zptCat 

679 

680 

681def makeAtmSchema(): 

682 """ 

683 Make the atmosphere schema 

684 

685 Returns 

686 ------- 

687 atmSchema: `lsst.afw.table.Schema` 

688 """ 

689 

690 atmSchema = afwTable.Schema() 

691 

692 atmSchema.addField('visit', type=np.int32, doc='Visit number') 

693 atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)') 

694 atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)') 

695 atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth') 

696 atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope') 

697 atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)') 

698 atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)') 

699 atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor') 

700 atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction') 

701 

702 return atmSchema 

703 

704 

705def makeAtmCat(atmSchema, atmStruct): 

706 """ 

707 Make the atmosphere catalog for persistence 

708 

709 Parameters 

710 ---------- 

711 atmSchema: `lsst.afw.table.Schema` 

712 Atmosphere catalog schema 

713 atmStruct: `numpy.ndarray` 

714 Atmosphere structure from fgcm 

715 

716 Returns 

717 ------- 

718 atmCat: `lsst.afw.table.BaseCatalog` 

719 Atmosphere catalog for persistence 

720 """ 

721 

722 atmCat = afwTable.BaseCatalog(atmSchema) 

723 atmCat.resize(atmStruct.size) 

724 

725 atmCat['visit'][:] = atmStruct['VISIT'] 

726 atmCat['pmb'][:] = atmStruct['PMB'] 

727 atmCat['pwv'][:] = atmStruct['PWV'] 

728 atmCat['tau'][:] = atmStruct['TAU'] 

729 atmCat['alpha'][:] = atmStruct['ALPHA'] 

730 atmCat['o3'][:] = atmStruct['O3'] 

731 atmCat['secZenith'][:] = atmStruct['SECZENITH'] 

732 atmCat['cTrans'][:] = atmStruct['CTRANS'] 

733 atmCat['lamStd'][:] = atmStruct['LAMSTD'] 

734 

735 return atmCat 

736 

737 

738def makeStdSchema(nBands): 

739 """ 

740 Make the standard star schema 

741 

742 Parameters 

743 ---------- 

744 nBands: `int` 

745 Number of bands in standard star catalog 

746 

747 Returns 

748 ------- 

749 stdSchema: `lsst.afw.table.Schema` 

750 """ 

751 

752 stdSchema = afwTable.SimpleTable.makeMinimalSchema() 

753 stdSchema.addField('ngood', type='ArrayI', doc='Number of good observations', 

754 size=nBands) 

755 stdSchema.addField('ntotal', type='ArrayI', doc='Number of total observations', 

756 size=nBands) 

757 stdSchema.addField('mag_std_noabs', type='ArrayF', 

758 doc='Standard magnitude (no absolute calibration)', 

759 size=nBands) 

760 stdSchema.addField('magErr_std', type='ArrayF', 

761 doc='Standard magnitude error', 

762 size=nBands) 

763 stdSchema.addField('npsfcand', type='ArrayI', 

764 doc='Number of observations flagged as psf candidates', 

765 size=nBands) 

766 

767 return stdSchema 

768 

769 

770def makeStdCat(stdSchema, stdStruct, goodBands): 

771 """ 

772 Make the standard star catalog for persistence 

773 

774 Parameters 

775 ---------- 

776 stdSchema: `lsst.afw.table.Schema` 

777 Standard star catalog schema 

778 stdStruct: `numpy.ndarray` 

779 Standard star structure in FGCM format 

780 goodBands: `list` 

781 List of good band names used in stdStruct 

782 

783 Returns 

784 ------- 

785 stdCat: `lsst.afw.table.BaseCatalog` 

786 Standard star catalog for persistence 

787 """ 

788 

789 stdCat = afwTable.SimpleCatalog(stdSchema) 

790 stdCat.resize(stdStruct.size) 

791 

792 stdCat['id'][:] = stdStruct['FGCM_ID'] 

793 stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees 

794 stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees 

795 stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :] 

796 stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :] 

797 stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :] 

798 stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :] 

799 stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :] 

800 

801 md = PropertyList() 

802 md.set("BANDS", list(goodBands)) 

803 stdCat.setMetadata(md) 

804 

805 return stdCat 

806 

807 

808def computeApertureRadiusFromDataRef(dataRef, fluxField): 

809 """ 

810 Compute the radius associated with a CircularApertureFlux field or 

811 associated slot. 

812 

813 Parameters 

814 ---------- 

815 dataRef : `lsst.daf.persistence.ButlerDataRef` or 

816 `lsst.daf.butler.DeferredDatasetHandle` 

817 fluxField : `str` 

818 CircularApertureFlux or associated slot. 

819 

820 Returns 

821 ------- 

822 apertureRadius : `float` 

823 Radius of the aperture field, in pixels. 

824 

825 Raises 

826 ------ 

827 RuntimeError: Raised if flux field is not a CircularApertureFlux, ApFlux, 

828 apFlux, or associated slot. 

829 """ 

830 # TODO: Move this method to more general stack method in DM-25775 

831 if isinstance(dataRef, dafPersist.ButlerDataRef): 

832 # Gen2 dataRef 

833 datasetType = dataRef.butlerSubset.datasetType 

834 else: 

835 # Gen3 dataRef 

836 datasetType = dataRef.ref.datasetType.name 

837 

838 if datasetType == 'src': 

839 schema = dataRef.get(datasetType='src_schema').schema 

840 try: 

841 fluxFieldName = schema[fluxField].asField().getName() 

842 except LookupError: 

843 raise RuntimeError("Could not find %s or associated slot in schema." % (fluxField)) 

844 # This may also raise a RuntimeError 

845 apertureRadius = computeApertureRadiusFromName(fluxFieldName) 

846 else: 

847 # This is a sourceTable_visit 

848 apertureRadius = computeApertureRadiusFromName(fluxField) 

849 

850 return apertureRadius 

851 

852 

853def computeApertureRadiusFromName(fluxField): 

854 """ 

855 Compute the radius associated with a CircularApertureFlux or ApFlux field. 

856 

857 Parameters 

858 ---------- 

859 fluxField : `str` 

860 CircularApertureFlux or ApFlux 

861 

862 Returns 

863 ------- 

864 apertureRadius : `float` 

865 Radius of the aperture field, in pixels. 

866 

867 Raises 

868 ------ 

869 RuntimeError: Raised if flux field is not a CircularApertureFlux, 

870 ApFlux, or apFlux. 

871 """ 

872 # TODO: Move this method to more general stack method in DM-25775 

873 m = re.search(r'(CircularApertureFlux|ApFlux|apFlux)_(\d+)_(\d+)_', fluxField) 

874 

875 if m is None: 

876 raise RuntimeError(f"Flux field {fluxField} does not correspond to a CircularApertureFlux or ApFlux") 

877 

878 apertureRadius = float(m.groups()[1]) + float(m.groups()[2])/10. 

879 

880 return apertureRadius 

881 

882 

883def extractReferenceMags(refStars, bands, filterMap): 

884 """ 

885 Extract reference magnitudes from refStars for given bands and 

886 associated filterMap. 

887 

888 Parameters 

889 ---------- 

890 refStars : `lsst.afw.table.BaseCatalog` 

891 FGCM reference star catalog 

892 bands : `list` 

893 List of bands for calibration 

894 filterMap: `dict` 

895 FGCM mapping of filter to band 

896 

897 Returns 

898 ------- 

899 refMag : `np.ndarray` 

900 nstar x nband array of reference magnitudes 

901 refMagErr : `np.ndarray` 

902 nstar x nband array of reference magnitude errors 

903 """ 

904 # After DM-23331 fgcm reference catalogs have FILTERNAMES to prevent 

905 # against index errors and allow more flexibility in fitting after 

906 # the build stars step. 

907 

908 md = refStars.getMetadata() 

909 if 'FILTERNAMES' in md: 

910 filternames = md.getArray('FILTERNAMES') 

911 

912 # The reference catalog that fgcm wants has one entry per band 

913 # in the config file 

914 refMag = np.zeros((len(refStars), len(bands)), 

915 dtype=refStars['refMag'].dtype) + 99.0 

916 refMagErr = np.zeros_like(refMag) + 99.0 

917 for i, filtername in enumerate(filternames): 

918 # We are allowed to run the fit configured so that we do not 

919 # use every column in the reference catalog. 

920 try: 

921 band = filterMap[filtername] 

922 except KeyError: 

923 continue 

924 try: 

925 ind = bands.index(band) 

926 except ValueError: 

927 continue 

928 

929 refMag[:, ind] = refStars['refMag'][:, i] 

930 refMagErr[:, ind] = refStars['refMagErr'][:, i] 

931 

932 else: 

933 # Continue to use old catalogs as before. 

934 refMag = refStars['refMag'][:, :] 

935 refMagErr = refStars['refMagErr'][:, :] 

936 

937 return refMag, refMagErr 

938 

939 

940def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

941 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

942 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

943 

944 return registry.queryDatasets(datasetType, 

945 dataId=quantumDataId, 

946 collections=[unboundedCollection])