Coverage for python/lsst/fgcmcal/fgcmCalibrateTractBase.py: 17%

153 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-29 03:46 -0700

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for running fgcmcal on a single tract using src tables 

22or sourceTable_visit tables. 

23""" 

24 

25import abc 

26 

27import numpy as np 

28 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31 

32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask 

33from .fgcmFitCycle import FgcmFitCycleConfig 

34from .fgcmOutputProducts import FgcmOutputProductsTask 

35from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

36from .utilities import computeApertureRadiusFromName, extractReferenceMags 

37from .utilities import makeZptSchema, makeZptCat 

38from .utilities import makeAtmSchema, makeAtmCat 

39from .utilities import makeStdSchema, makeStdCat 

40from .focalPlaneProjector import FocalPlaneProjector 

41 

42import fgcm 

43 

44__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask'] 

45 

46 

47class FgcmCalibrateTractConfigBase(pexConfig.Config): 

48 """Config for FgcmCalibrateTract""" 

49 

50 fgcmBuildStars = pexConfig.ConfigurableField( 

51 target=FgcmBuildStarsTableTask, 

52 doc="Task to load and match stars for fgcm", 

53 ) 

54 fgcmFitCycle = pexConfig.ConfigField( 

55 dtype=FgcmFitCycleConfig, 

56 doc="Config to run a single fgcm fit cycle", 

57 ) 

58 fgcmOutputProducts = pexConfig.ConfigurableField( 

59 target=FgcmOutputProductsTask, 

60 doc="Task to output fgcm products", 

61 ) 

62 convergenceTolerance = pexConfig.Field( 

63 doc="Tolerance on repeatability convergence (per band)", 

64 dtype=float, 

65 default=0.005, 

66 ) 

67 maxFitCycles = pexConfig.Field( 

68 doc="Maximum number of fit cycles", 

69 dtype=int, 

70 default=5, 

71 ) 

72 doDebuggingPlots = pexConfig.Field( 

73 doc="Make plots for debugging purposes?", 

74 dtype=bool, 

75 default=False, 

76 ) 

77 

78 def setDefaults(self): 

79 pexConfig.Config.setDefaults(self) 

80 

81 self.fgcmFitCycle.quietMode = True 

82 self.fgcmFitCycle.doPlots = False 

83 self.fgcmOutputProducts.doReferenceCalibration = False 

84 self.fgcmOutputProducts.cycleNumber = 0 

85 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

86 

87 def validate(self): 

88 super().validate() 

89 

90 for band in self.fgcmFitCycle.bands: 

91 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]: 

92 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 

93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

94 self, msg) 

95 

96 

97class FgcmCalibrateTractBaseTask(pipeBase.PipelineTask, abc.ABC): 

98 """Base class to calibrate a single tract using fgcmcal 

99 """ 

100 def __init__(self, initInputs=None, **kwargs): 

101 super().__init__(**kwargs) 

102 self.makeSubtask("fgcmBuildStars", initInputs=initInputs) 

103 self.makeSubtask("fgcmOutputProducts") 

104 

105 def run(self, handleDict, tract, 

106 buildStarsRefObjLoader=None, returnCatalogs=True): 

107 """Run the calibrations for a single tract with fgcm. 

108 

109 Parameters 

110 ---------- 

111 handleDict : `dict` 

112 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

113 handle dictionary with the following keys. Note that all 

114 keys need not be set based on config parameters. 

115 

116 ``"camera"`` 

117 Camera object (`lsst.afw.cameraGeom.Camera`) 

118 ``"source_catalogs"`` 

119 `list` of handles for input source catalogs. 

120 ``"sourceSchema"`` 

121 Schema for the source catalogs. 

122 ``"fgcmLookUpTable"`` 

123 handle for the FGCM look-up table. 

124 ``"calexps"`` 

125 `list` of handles for the input calexps 

126 ``"fgcmPhotoCalibs"`` 

127 `dict` of output photoCalib handles. Key is 

128 (tract, visit, detector). 

129 Present if doZeropointOutput is True. 

130 ``"fgcmTransmissionAtmospheres"`` 

131 `dict` of output atmosphere transmission handles. 

132 Key is (tract, visit). 

133 Present if doAtmosphereOutput is True. 

134 tract : `int` 

135 Tract number 

136 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

137 Reference object loader object for fgcmBuildStars. 

138 returnCatalogs : `bool`, optional 

139 Return photoCalibs as per-visit exposure catalogs. 

140 

141 Returns 

142 ------- 

143 outstruct : `lsst.pipe.base.Struct` 

144 Output structure with keys: 

145 

146 offsets : `np.ndarray` 

147 Final reference offsets, per band. 

148 repeatability : `np.ndarray` 

149 Raw fgcm repeatability for bright stars, per band. 

150 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

151 Generator that returns (visit, transmissionCurve) tuples. 

152 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

153 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

154 (returned if returnCatalogs is False). 

155 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

156 Generator that returns (visit, exposureCatalog) tuples. 

157 (returned if returnCatalogs is True). 

158 """ 

159 self.log.info("Running on tract %d", (tract)) 

160 

161 # Compute the aperture radius if necessary. This is useful to do now before 

162 # any heavy lifting has happened (fail early). 

163 calibFluxApertureRadius = None 

164 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

165 try: 

166 field = self.config.fgcmBuildStars.instFluxField 

167 calibFluxApertureRadius = computeApertureRadiusFromName(field) 

168 except RuntimeError: 

169 raise RuntimeError("Could not determine aperture radius from %s. " 

170 "Cannot use doSubtractLocalBackground." % 

171 (field)) 

172 

173 # Run the build stars tasks 

174 

175 # Note that we will need visitCat at the end of the procedure for the outputs 

176 groupedHandles = self.fgcmBuildStars._groupHandles(handleDict['sourceTableHandleDict'], 

177 handleDict['visitSummaryHandleDict']) 

178 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(handleDict['camera'], groupedHandles) 

179 rad = calibFluxApertureRadius 

180 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedHandles, 

181 visitCat, 

182 handleDict['sourceSchema'], 

183 handleDict['camera'], 

184 calibFluxApertureRadius=rad) 

185 

186 if self.fgcmBuildStars.config.doReferenceMatches: 

187 lutHandle = handleDict['fgcmLookUpTable'] 

188 self.fgcmBuildStars.makeSubtask("fgcmLoadReferenceCatalog", 

189 refObjLoader=buildStarsRefObjLoader, 

190 refCatName=self.fgcmBuildStars.config.connections.refCat) 

191 else: 

192 lutHandle = None 

193 

194 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

195 self.fgcmBuildStars.fgcmMatchStars(visitCat, 

196 fgcmStarObservationCat, 

197 lutHandle=lutHandle) 

198 

199 # Load the LUT 

200 lutCat = handleDict['fgcmLookUpTable'].get() 

201 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

202 dict(self.config.fgcmFitCycle.physicalFilterMap)) 

203 del lutCat 

204 

205 # Translate the visit catalog into fgcm format 

206 fgcmExpInfo = translateVisitCatalog(visitCat) 

207 

208 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, handleDict['camera'], 

209 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

210 True, False, lutIndexVals[0]['FILTERNAMES'], 

211 tract=tract) 

212 

213 focalPlaneProjector = FocalPlaneProjector(handleDict['camera'], 

214 self.config.fgcmFitCycle.defaultCameraOrientation) 

215 

216 # Set up the fit cycle task 

217 

218 noFitsDict = {'lutIndex': lutIndexVals, 

219 'lutStd': lutStd, 

220 'expInfo': fgcmExpInfo, 

221 'focalPlaneProjector': focalPlaneProjector} 

222 

223 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

224 noFitsDict=noFitsDict, noOutput=True) 

225 

226 # We determine the conversion from the native units (typically radians) to 

227 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

228 # numpy arrays rather than Angles, which would we approximately 600x slower. 

229 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

230 

231 # To load the stars, we need an initial parameter object 

232 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

233 fgcmLut, 

234 fgcmExpInfo) 

235 

236 # Match star observations to visits 

237 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

238 # actually be transferred into fgcm using the indexing below. 

239 

240 obsIndex = fgcmStarIndicesCat['obsIndex'] 

241 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

242 fgcmStarObservationCat['visit'][obsIndex]) 

243 

244 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

245 self.config.fgcmFitCycle.bands, 

246 self.config.fgcmFitCycle.physicalFilterMap) 

247 refId = fgcmRefCat['fgcm_id'][:] 

248 

249 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

250 fgcmStars.loadStars(fgcmPars, 

251 fgcmStarObservationCat['visit'][obsIndex], 

252 fgcmStarObservationCat['ccd'][obsIndex], 

253 fgcmStarObservationCat['ra'][obsIndex] * conv, 

254 fgcmStarObservationCat['dec'][obsIndex] * conv, 

255 fgcmStarObservationCat['instMag'][obsIndex], 

256 fgcmStarObservationCat['instMagErr'][obsIndex], 

257 fgcmExpInfo['FILTERNAME'][visitIndex], 

258 fgcmStarIdCat['fgcm_id'][:], 

259 fgcmStarIdCat['ra'][:], 

260 fgcmStarIdCat['dec'][:], 

261 fgcmStarIdCat['obsArrIndex'][:], 

262 fgcmStarIdCat['nObs'][:], 

263 obsX=fgcmStarObservationCat['x'][obsIndex], 

264 obsY=fgcmStarObservationCat['y'][obsIndex], 

265 obsDeltaMagBkg=fgcmStarObservationCat['deltaMagBkg'][obsIndex], 

266 obsDeltaAper=fgcmStarObservationCat['deltaMagAper'][obsIndex], 

267 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

268 refID=refId, 

269 refMag=refMag, 

270 refMagErr=refMagErr, 

271 flagID=None, 

272 flagFlag=None, 

273 computeNobs=True) 

274 

275 # Clear out some memory 

276 del fgcmStarIdCat 

277 del fgcmStarIndicesCat 

278 del fgcmRefCat 

279 

280 fgcmFitCycle.setLUT(fgcmLut) 

281 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

282 

283 converged = False 

284 cycleNumber = 0 

285 

286 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

287 previousParInfo = None 

288 previousParams = None 

289 previousSuperStar = None 

290 

291 while (not converged and cycleNumber < self.config.maxFitCycles): 

292 

293 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

294 

295 if cycleNumber > 0: 

296 # Use parameters from previous cycle 

297 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

298 fgcmExpInfo, 

299 previousParInfo, 

300 previousParams, 

301 previousSuperStar) 

302 # We need to reset the star magnitudes and errors for the next 

303 # cycle 

304 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

305 fgcmStarObservationCat['instMagErr'][obsIndex]) 

306 fgcmFitCycle.initialCycle = False 

307 

308 fgcmFitCycle.setPars(fgcmPars) 

309 fgcmFitCycle.finishSetup() 

310 

311 fgcmFitCycle.run() 

312 

313 # Grab the parameters for the next cycle 

314 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

315 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

316 

317 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

318 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

319 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

320 continue 

321 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

322 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

323 

324 # Check for convergence 

325 if np.alltrue((previousReservedRawRepeatability 

326 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability) 

327 < self.config.convergenceTolerance): 

328 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

329 converged = True 

330 else: 

331 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

332 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

333 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

334 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

335 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

336 self.log.info("Setting exposure gray photometricity cuts to:") 

337 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

338 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

339 continue 

340 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

341 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

342 

343 cycleNumber += 1 

344 

345 # Log warning if not converged 

346 if not converged: 

347 self.log.warning("Maximum number of fit cycles exceeded (%d) without convergence.", cycleNumber) 

348 

349 # Do final clean-up iteration 

350 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

351 fgcmFitCycle.fgcmConfig.resetParameters = False 

352 fgcmFitCycle.fgcmConfig.maxIter = 0 

353 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

354 fgcmFitCycle.fgcmConfig.outputStandards = True 

355 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

356 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

357 fgcmFitCycle.initialCycle = False 

358 

359 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

360 fgcmExpInfo, 

361 previousParInfo, 

362 previousParams, 

363 previousSuperStar) 

364 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

365 fgcmStarObservationCat['instMagErr'][obsIndex]) 

366 fgcmFitCycle.setPars(fgcmPars) 

367 fgcmFitCycle.finishSetup() 

368 

369 self.log.info("Running final clean-up fit cycle...") 

370 fgcmFitCycle.run() 

371 

372 self.log.info("Raw repeatability after clean-up cycle is:") 

373 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

374 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

375 continue 

376 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

377 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

378 

379 # Do the outputs. Need to keep track of tract. 

380 

381 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

382 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

383 

384 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

385 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

386 

387 atmSchema = makeAtmSchema() 

388 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

389 

390 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

391 stdSchema = makeStdSchema(len(goodBands)) 

392 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

393 

394 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(handleDict, 

395 tract, 

396 visitCat, 

397 zptCat, atmCat, stdCat, 

398 self.config.fgcmBuildStars) 

399 

400 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

401 

402 fgcmFitCycle.freeSharedMemory() 

403 

404 return outStruct