Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for running fgcmcal on a single tract using src tables 

22or sourceTable_visit tables. 

23""" 

24 

25import sys 

26import traceback 

27 

28import numpy as np 

29 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32 

33from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig 

34from .fgcmFitCycle import FgcmFitCycleConfig 

35from .fgcmOutputProducts import FgcmOutputProductsTask 

36from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

37from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags 

38from .utilities import makeZptSchema, makeZptCat 

39from .utilities import makeAtmSchema, makeAtmCat 

40from .utilities import makeStdSchema, makeStdCat 

41 

42import fgcm 

43 

44__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner'] 

45 

46 

47class FgcmCalibrateTractConfigBase(pexConfig.Config): 

48 """Config for FgcmCalibrateTract""" 

49 

50 fgcmBuildStars = pexConfig.ConfigurableField( 

51 target=FgcmBuildStarsTask, 

52 doc="Task to load and match stars for fgcm", 

53 ) 

54 fgcmFitCycle = pexConfig.ConfigField( 

55 dtype=FgcmFitCycleConfig, 

56 doc="Config to run a single fgcm fit cycle", 

57 ) 

58 fgcmOutputProducts = pexConfig.ConfigurableField( 

59 target=FgcmOutputProductsTask, 

60 doc="Task to output fgcm products", 

61 ) 

62 convergenceTolerance = pexConfig.Field( 

63 doc="Tolerance on repeatability convergence (per band)", 

64 dtype=float, 

65 default=0.005, 

66 ) 

67 maxFitCycles = pexConfig.Field( 

68 doc="Maximum number of fit cycles", 

69 dtype=int, 

70 default=5, 

71 ) 

72 doDebuggingPlots = pexConfig.Field( 

73 doc="Make plots for debugging purposes?", 

74 dtype=bool, 

75 default=False, 

76 ) 

77 

78 def setDefaults(self): 

79 pexConfig.Config.setDefaults(self) 

80 

81 self.fgcmFitCycle.quietMode = True 

82 self.fgcmOutputProducts.doReferenceCalibration = False 

83 self.fgcmOutputProducts.doRefcatOutput = False 

84 self.fgcmOutputProducts.cycleNumber = 0 

85 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

86 

87 def validate(self): 

88 super().validate() 

89 

90 for band in self.fgcmFitCycle.bands: 

91 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]: 

92 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 

93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

94 self, msg) 

95 

96 

97class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner): 

98 """Subclass of TaskRunner for FgcmCalibrateTractTask 

99 

100 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 

101 the butler (for persistence and mapper data), and a list of dataRefs 

102 extracted from the command line. This task runs on a constrained set 

103 of dataRefs, typically a single tract. 

104 This class transforms the process arguments generated by the ArgumentParser 

105 into the arguments expected by FgcmCalibrateTractTask.run(). 

106 This runner does not use any parallelization. 

107 """ 

108 

109 @staticmethod 

110 def getTargetList(parsedCmd): 

111 """ 

112 Return a list with one element: a tuple with the butler and 

113 list of dataRefs. 

114 """ 

115 return [(parsedCmd.butler, parsedCmd.id.refList)] 

116 

117 def __call__(self, args): 

118 """ 

119 Parameters 

120 ---------- 

121 args: `tuple` with (butler, dataRefList) 

122 

123 Returns 

124 ------- 

125 exitStatus: `list` with `lsst.pipe.base.Struct` 

126 exitStatus (0: success; 1: failure) 

127 May also contain results if `self.doReturnResults` is `True`. 

128 """ 

129 butler, dataRefList = args 

130 

131 task = self.TaskClass(config=self.config, log=self.log) 

132 

133 exitStatus = 0 

134 if self.doRaise: 

135 results = task.runDataRef(butler, dataRefList) 

136 else: 

137 try: 

138 results = task.runDataRef(butler, dataRefList) 

139 except Exception as e: 

140 exitStatus = 1 

141 task.log.fatal("Failed: %s" % e) 

142 if not isinstance(e, pipeBase.TaskError): 

143 traceback.print_exc(file=sys.stderr) 

144 

145 task.writeMetadata(butler) 

146 

147 if self.doReturnResults: 

148 return [pipeBase.Struct(exitStatus=exitStatus, 

149 results=results)] 

150 else: 

151 return [pipeBase.Struct(exitStatus=exitStatus)] 

152 

153 def run(self, parsedCmd): 

154 """ 

155 Run the task, with no multiprocessing 

156 

157 Parameters 

158 ---------- 

159 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

160 """ 

161 

162 resultList = [] 

163 

164 if self.precall(parsedCmd): 

165 targetList = self.getTargetList(parsedCmd) 

166 resultList = self(targetList[0]) 

167 

168 return resultList 

169 

170 

171class FgcmCalibrateTractBaseTask(pipeBase.CmdLineTask): 

172 """ 

173 Base class to calibrate a single tract using fgcmcal 

174 """ 

175 def __init__(self, butler=None, **kwargs): 

176 """ 

177 Instantiate an `FgcmCalibrateTractTask`. 

178 

179 Parameters 

180 ---------- 

181 butler : `lsst.daf.persistence.Butler` 

182 """ 

183 

184 pipeBase.CmdLineTask.__init__(self, **kwargs) 

185 

186 # no saving of metadata for now 

187 def _getMetadataName(self): 

188 return None 

189 

190 @pipeBase.timeMethod 

191 def runDataRef(self, butler, dataRefs): 

192 """ 

193 Run full FGCM calibration on a single tract, including building star list, 

194 fitting multiple cycles, and making outputs. 

195 

196 Parameters 

197 ---------- 

198 butler: `lsst.daf.persistence.Butler` 

199 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

200 Data references for the input visits. 

201 These may be either per-ccd "src" or per-visit"sourceTable_visit" 

202 references. 

203 

204 Raises 

205 ------ 

206 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 

207 not True, or if fgcmLookUpTable is not available, or if 

208 doSubtractLocalBackground is True and aperture radius cannot be 

209 determined. 

210 """ 

211 datasetType = dataRefs[0].butlerSubset.datasetType 

212 self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType)) 

213 

214 if not butler.datasetExists('fgcmLookUpTable'): 

215 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable") 

216 

217 if not self.config.fgcmBuildStars.doReferenceMatches: 

218 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches") 

219 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig): 

220 if self.config.fgcmBuildStars.checkAllCcds: 

221 raise RuntimeError("Cannot run FgcmCalibrateTract with " 

222 "fgcmBuildStars.checkAllCcds set to True") 

223 

224 self.makeSubtask("fgcmBuildStars", butler=butler) 

225 self.makeSubtask("fgcmOutputProducts", butler=butler) 

226 

227 # Compute the aperture radius if necessary. This is useful to do now before 

228 # any heavy lifting has happened (fail early). 

229 calibFluxApertureRadius = None 

230 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

231 try: 

232 field = self.config.fgcmBuildStars.instFluxField 

233 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

234 field) 

235 except RuntimeError: 

236 raise RuntimeError("Could not determine aperture radius from %s. " 

237 "Cannot use doSubtractLocalBackground." % 

238 (field)) 

239 

240 # Run the build stars tasks 

241 tract = int(dataRefs[0].dataId['tract']) 

242 self.log.info("Running on tract %d" % (tract)) 

243 

244 # Note that we will need visitCat at the end of the procedure for the outputs 

245 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs) 

246 camera = butler.get('camera') 

247 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs) 

248 rad = calibFluxApertureRadius 

249 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs, 

250 visitCat, 

251 calibFluxApertureRadius=rad) 

252 

253 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

254 self.fgcmBuildStars.fgcmMatchStars(butler, 

255 visitCat, 

256 fgcmStarObservationCat) 

257 

258 # Load the LUT 

259 lutCat = butler.get('fgcmLookUpTable') 

260 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

261 dict(self.config.fgcmFitCycle.filterMap)) 

262 del lutCat 

263 

264 # Translate the visit catalog into fgcm format 

265 fgcmExpInfo = translateVisitCatalog(visitCat) 

266 

267 camera = butler.get('camera') 

268 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera, 

269 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

270 True, False, tract=tract) 

271 # Turn off plotting in tract mode 

272 configDict['doPlots'] = False 

273 

274 # Use the first orientation. 

275 # TODO: DM-21215 will generalize to arbitrary camera orientations 

276 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

277 del camera 

278 

279 # Set up the fit cycle task 

280 

281 noFitsDict = {'lutIndex': lutIndexVals, 

282 'lutStd': lutStd, 

283 'expInfo': fgcmExpInfo, 

284 'ccdOffsets': ccdOffsets} 

285 

286 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

287 noFitsDict=noFitsDict, noOutput=True) 

288 

289 # We determine the conversion from the native units (typically radians) to 

290 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

291 # numpy arrays rather than Angles, which would we approximately 600x slower. 

292 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

293 

294 # To load the stars, we need an initial parameter object 

295 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

296 fgcmLut, 

297 fgcmExpInfo) 

298 

299 # Match star observations to visits 

300 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

301 # actually be transferred into fgcm using the indexing below. 

302 

303 obsIndex = fgcmStarIndicesCat['obsIndex'] 

304 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

305 fgcmStarObservationCat['visit'][obsIndex]) 

306 

307 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

308 self.config.fgcmFitCycle.bands, 

309 self.config.fgcmFitCycle.filterMap) 

310 refId = fgcmRefCat['fgcm_id'][:] 

311 

312 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

313 fgcmStars.loadStars(fgcmPars, 

314 fgcmStarObservationCat['visit'][obsIndex], 

315 fgcmStarObservationCat['ccd'][obsIndex], 

316 fgcmStarObservationCat['ra'][obsIndex] * conv, 

317 fgcmStarObservationCat['dec'][obsIndex] * conv, 

318 fgcmStarObservationCat['instMag'][obsIndex], 

319 fgcmStarObservationCat['instMagErr'][obsIndex], 

320 fgcmExpInfo['FILTERNAME'][visitIndex], 

321 fgcmStarIdCat['fgcm_id'][:], 

322 fgcmStarIdCat['ra'][:], 

323 fgcmStarIdCat['dec'][:], 

324 fgcmStarIdCat['obsArrIndex'][:], 

325 fgcmStarIdCat['nObs'][:], 

326 obsX=fgcmStarObservationCat['x'][obsIndex], 

327 obsY=fgcmStarObservationCat['y'][obsIndex], 

328 obsDeltaMagBkg=fgcmStarObservationCat['deltaMagBkg'][obsIndex], 

329 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

330 refID=refId, 

331 refMag=refMag, 

332 refMagErr=refMagErr, 

333 flagID=None, 

334 flagFlag=None, 

335 computeNobs=True) 

336 

337 # Clear out some memory 

338 del fgcmStarIdCat 

339 del fgcmStarIndicesCat 

340 del fgcmRefCat 

341 

342 fgcmFitCycle.setLUT(fgcmLut) 

343 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

344 

345 converged = False 

346 cycleNumber = 0 

347 

348 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

349 previousParInfo = None 

350 previousParams = None 

351 previousSuperStar = None 

352 

353 while (not converged and cycleNumber < self.config.maxFitCycles): 

354 

355 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

356 

357 if cycleNumber > 0: 

358 # Use parameters from previous cycle 

359 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

360 fgcmExpInfo, 

361 previousParInfo, 

362 previousParams, 

363 previousSuperStar) 

364 # We need to reset the star magnitudes and errors for the next 

365 # cycle 

366 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

367 fgcmStarObservationCat['instMagErr'][obsIndex]) 

368 fgcmFitCycle.initialCycle = False 

369 

370 fgcmFitCycle.setPars(fgcmPars) 

371 fgcmFitCycle.finishSetup() 

372 

373 fgcmFitCycle.run() 

374 

375 # Grab the parameters for the next cycle 

376 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

377 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

378 

379 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

380 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

381 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

382 continue 

383 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

384 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

385 

386 # Check for convergence 

387 if np.alltrue((previousReservedRawRepeatability - 

388 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) < 

389 self.config.convergenceTolerance): 

390 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

391 converged = True 

392 else: 

393 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

394 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

395 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

396 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

397 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

398 self.log.info("Setting exposure gray photometricity cuts to:") 

399 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

400 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

401 continue 

402 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

403 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

404 

405 cycleNumber += 1 

406 

407 # Log warning if not converged 

408 if not converged: 

409 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber)) 

410 

411 # Do final clean-up iteration 

412 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

413 fgcmFitCycle.fgcmConfig.resetParameters = False 

414 fgcmFitCycle.fgcmConfig.maxIter = 0 

415 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

416 fgcmFitCycle.fgcmConfig.outputStandards = True 

417 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

418 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

419 fgcmFitCycle.initialCycle = False 

420 

421 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

422 fgcmExpInfo, 

423 previousParInfo, 

424 previousParams, 

425 previousSuperStar) 

426 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

427 fgcmStarObservationCat['instMagErr'][obsIndex]) 

428 fgcmFitCycle.setPars(fgcmPars) 

429 fgcmFitCycle.finishSetup() 

430 

431 self.log.info("Running final clean-up fit cycle...") 

432 fgcmFitCycle.run() 

433 

434 self.log.info("Raw repeatability after clean-up cycle is:") 

435 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

436 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

437 continue 

438 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

439 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

440 

441 # Do the outputs. Need to keep track of tract. 

442 

443 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

444 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

445 

446 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

447 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

448 

449 atmSchema = makeAtmSchema() 

450 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

451 

452 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

453 stdSchema = makeStdSchema(len(goodBands)) 

454 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

455 

456 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract, 

457 visitCat, 

458 zptCat, atmCat, stdCat, 

459 self.config.fgcmBuildStars, 

460 self.config.fgcmFitCycle) 

461 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

462 

463 return outStruct