Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for running fgcmcal on a single tract using src tables 

22or sourceTable_visit tables. 

23""" 

24 

25import sys 

26import traceback 

27 

28import numpy as np 

29 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32 

33from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig 

34from .fgcmFitCycle import FgcmFitCycleConfig 

35from .fgcmOutputProducts import FgcmOutputProductsTask 

36from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

37from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags 

38from .utilities import makeZptSchema, makeZptCat 

39from .utilities import makeAtmSchema, makeAtmCat 

40from .utilities import makeStdSchema, makeStdCat 

41 

42import fgcm 

43 

44__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner'] 

45 

46 

47class FgcmCalibrateTractConfigBase(pexConfig.Config): 

48 """Config for FgcmCalibrateTract""" 

49 

50 fgcmBuildStars = pexConfig.ConfigurableField( 

51 target=FgcmBuildStarsTask, 

52 doc="Task to load and match stars for fgcm", 

53 ) 

54 fgcmFitCycle = pexConfig.ConfigField( 

55 dtype=FgcmFitCycleConfig, 

56 doc="Config to run a single fgcm fit cycle", 

57 ) 

58 fgcmOutputProducts = pexConfig.ConfigurableField( 

59 target=FgcmOutputProductsTask, 

60 doc="Task to output fgcm products", 

61 ) 

62 convergenceTolerance = pexConfig.Field( 

63 doc="Tolerance on repeatability convergence (per band)", 

64 dtype=float, 

65 default=0.005, 

66 ) 

67 maxFitCycles = pexConfig.Field( 

68 doc="Maximum number of fit cycles", 

69 dtype=int, 

70 default=5, 

71 ) 

72 doDebuggingPlots = pexConfig.Field( 

73 doc="Make plots for debugging purposes?", 

74 dtype=bool, 

75 default=False, 

76 ) 

77 

78 def setDefaults(self): 

79 pexConfig.Config.setDefaults(self) 

80 

81 self.fgcmFitCycle.quietMode = True 

82 self.fgcmOutputProducts.doReferenceCalibration = False 

83 self.fgcmOutputProducts.doRefcatOutput = False 

84 self.fgcmOutputProducts.cycleNumber = 0 

85 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

86 

87 def validate(self): 

88 super().validate() 

89 

90 for band in self.fgcmFitCycle.bands: 

91 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]: 

92 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 

93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

94 self, msg) 

95 

96 

97class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner): 

98 """Subclass of TaskRunner for FgcmCalibrateTractTask 

99 

100 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 

101 the butler (for persistence and mapper data), and a list of dataRefs 

102 extracted from the command line. This task runs on a constrained set 

103 of dataRefs, typically a single tract. 

104 This class transforms the process arguments generated by the ArgumentParser 

105 into the arguments expected by FgcmCalibrateTractTask.run(). 

106 This runner does not use any parallelization. 

107 """ 

108 

109 @staticmethod 

110 def getTargetList(parsedCmd): 

111 """ 

112 Return a list with one element: a tuple with the butler and 

113 list of dataRefs. 

114 """ 

115 return [(parsedCmd.butler, parsedCmd.id.refList)] 

116 

117 def __call__(self, args): 

118 """ 

119 Parameters 

120 ---------- 

121 args: `tuple` with (butler, dataRefList) 

122 

123 Returns 

124 ------- 

125 exitStatus: `list` with `lsst.pipe.base.Struct` 

126 exitStatus (0: success; 1: failure) 

127 May also contain results if `self.doReturnResults` is `True`. 

128 """ 

129 butler, dataRefList = args 

130 

131 task = self.TaskClass(config=self.config, log=self.log) 

132 

133 exitStatus = 0 

134 if self.doRaise: 

135 results = task.runDataRef(butler, dataRefList) 

136 else: 

137 try: 

138 results = task.runDataRef(butler, dataRefList) 

139 except Exception as e: 

140 exitStatus = 1 

141 task.log.fatal("Failed: %s" % e) 

142 if not isinstance(e, pipeBase.TaskError): 

143 traceback.print_exc(file=sys.stderr) 

144 

145 task.writeMetadata(butler) 

146 

147 if self.doReturnResults: 

148 return [pipeBase.Struct(exitStatus=exitStatus, 

149 results=results)] 

150 else: 

151 return [pipeBase.Struct(exitStatus=exitStatus)] 

152 

153 def run(self, parsedCmd): 

154 """ 

155 Run the task, with no multiprocessing 

156 

157 Parameters 

158 ---------- 

159 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

160 """ 

161 

162 resultList = [] 

163 

164 if self.precall(parsedCmd): 

165 targetList = self.getTargetList(parsedCmd) 

166 resultList = self(targetList[0]) 

167 

168 return resultList 

169 

170 

171class FgcmCalibrateTractBaseTask(pipeBase.CmdLineTask): 

172 """ 

173 Base class to calibrate a single tract using fgcmcal 

174 """ 

175 def __init__(self, butler=None, **kwargs): 

176 """ 

177 Instantiate an `FgcmCalibrateTractTask`. 

178 

179 Parameters 

180 ---------- 

181 butler : `lsst.daf.persistence.Butler` 

182 """ 

183 

184 pipeBase.CmdLineTask.__init__(self, **kwargs) 

185 

186 # no saving of metadata for now 

187 def _getMetadataName(self): 

188 return None 

189 

190 @pipeBase.timeMethod 

191 def runDataRef(self, butler, dataRefs): 

192 """ 

193 Run full FGCM calibration on a single tract, including building star list, 

194 fitting multiple cycles, and making outputs. 

195 

196 Parameters 

197 ---------- 

198 butler: `lsst.daf.persistence.Butler` 

199 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

200 Data references for the input visits. 

201 These may be either per-ccd "src" or per-visit"sourceTable_visit" 

202 references. 

203 

204 Raises 

205 ------ 

206 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 

207 not True, or if fgcmLookUpTable is not available, or if 

208 doSubtractLocalBackground is True and aperture radius cannot be 

209 determined. 

210 """ 

211 datasetType = dataRefs[0].butlerSubset.datasetType 

212 self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType)) 

213 

214 if not butler.datasetExists('fgcmLookUpTable'): 

215 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable") 

216 

217 if not self.config.fgcmBuildStars.doReferenceMatches: 

218 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches") 

219 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig): 

220 if self.config.fgcmBuildStars.checkAllCcds: 

221 raise RuntimeError("Cannot run FgcmCalibrateTract with " 

222 "fgcmBuildStars.checkAllCcds set to True") 

223 

224 self.makeSubtask("fgcmBuildStars", butler=butler) 

225 self.makeSubtask("fgcmOutputProducts", butler=butler) 

226 

227 # Compute the aperture radius if necessary. This is useful to do now before 

228 # any heavy lifting has happened (fail early). 

229 calibFluxApertureRadius = None 

230 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

231 try: 

232 field = self.config.fgcmBuildStars.instFluxField 

233 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

234 field) 

235 except RuntimeError: 

236 raise RuntimeError("Could not determine aperture radius from %s. " 

237 "Cannot use doSubtractLocalBackground." % 

238 (field)) 

239 

240 # Run the build stars tasks 

241 tract = int(dataRefs[0].dataId['tract']) 

242 self.log.info("Running on tract %d" % (tract)) 

243 

244 # Note that we will need visitCat at the end of the procedure for the outputs 

245 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs) 

246 camera = butler.get('camera') 

247 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs) 

248 rad = calibFluxApertureRadius 

249 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs, 

250 visitCat, 

251 calibFluxApertureRadius=rad) 

252 

253 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

254 self.fgcmBuildStars.fgcmMatchStars(butler, 

255 visitCat, 

256 fgcmStarObservationCat) 

257 

258 # Load the LUT 

259 lutCat = butler.get('fgcmLookUpTable') 

260 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

261 dict(self.config.fgcmFitCycle.filterMap)) 

262 del lutCat 

263 

264 # Translate the visit catalog into fgcm format 

265 fgcmExpInfo = translateVisitCatalog(visitCat) 

266 

267 camera = butler.get('camera') 

268 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera, 

269 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

270 True, False, tract=tract) 

271 # Turn off plotting in tract mode 

272 configDict['doPlots'] = False 

273 

274 # Use the first orientation. 

275 # TODO: DM-21215 will generalize to arbitrary camera orientations 

276 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

277 del camera 

278 

279 # Set up the fit cycle task 

280 

281 noFitsDict = {'lutIndex': lutIndexVals, 

282 'lutStd': lutStd, 

283 'expInfo': fgcmExpInfo, 

284 'ccdOffsets': ccdOffsets} 

285 

286 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

287 noFitsDict=noFitsDict, noOutput=True) 

288 

289 # We determine the conversion from the native units (typically radians) to 

290 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

291 # numpy arrays rather than Angles, which would we approximately 600x slower. 

292 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

293 

294 # To load the stars, we need an initial parameter object 

295 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

296 fgcmLut, 

297 fgcmExpInfo) 

298 

299 # Match star observations to visits 

300 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

301 # actually be transferred into fgcm using the indexing below. 

302 

303 obsIndex = fgcmStarIndicesCat['obsIndex'] 

304 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

305 fgcmStarObservationCat['visit'][obsIndex]) 

306 

307 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

308 self.config.fgcmFitCycle.bands, 

309 self.config.fgcmFitCycle.filterMap) 

310 refId = fgcmRefCat['fgcm_id'][:] 

311 

312 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

313 fgcmStars.loadStars(fgcmPars, 

314 fgcmStarObservationCat['visit'][obsIndex], 

315 fgcmStarObservationCat['ccd'][obsIndex], 

316 fgcmStarObservationCat['ra'][obsIndex] * conv, 

317 fgcmStarObservationCat['dec'][obsIndex] * conv, 

318 fgcmStarObservationCat['instMag'][obsIndex], 

319 fgcmStarObservationCat['instMagErr'][obsIndex], 

320 fgcmExpInfo['FILTERNAME'][visitIndex], 

321 fgcmStarIdCat['fgcm_id'][:], 

322 fgcmStarIdCat['ra'][:], 

323 fgcmStarIdCat['dec'][:], 

324 fgcmStarIdCat['obsArrIndex'][:], 

325 fgcmStarIdCat['nObs'][:], 

326 obsX=fgcmStarObservationCat['x'][obsIndex], 

327 obsY=fgcmStarObservationCat['y'][obsIndex], 

328 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

329 refID=refId, 

330 refMag=refMag, 

331 refMagErr=refMagErr, 

332 flagID=None, 

333 flagFlag=None, 

334 computeNobs=True) 

335 

336 # Clear out some memory 

337 del fgcmStarIdCat 

338 del fgcmStarIndicesCat 

339 del fgcmRefCat 

340 

341 fgcmFitCycle.setLUT(fgcmLut) 

342 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

343 

344 converged = False 

345 cycleNumber = 0 

346 

347 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

348 previousParInfo = None 

349 previousParams = None 

350 previousSuperStar = None 

351 

352 while (not converged and cycleNumber < self.config.maxFitCycles): 

353 

354 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

355 

356 if cycleNumber > 0: 

357 # Use parameters from previous cycle 

358 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

359 fgcmExpInfo, 

360 previousParInfo, 

361 previousParams, 

362 previousSuperStar) 

363 # We need to reset the star magnitudes and errors for the next 

364 # cycle 

365 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

366 fgcmStarObservationCat['instMagErr'][obsIndex]) 

367 fgcmFitCycle.initialCycle = False 

368 

369 fgcmFitCycle.setPars(fgcmPars) 

370 fgcmFitCycle.finishSetup() 

371 

372 fgcmFitCycle.run() 

373 

374 # Grab the parameters for the next cycle 

375 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

376 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

377 

378 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

379 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

380 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

381 continue 

382 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

383 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

384 

385 # Check for convergence 

386 if np.alltrue((previousReservedRawRepeatability - 

387 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) < 

388 self.config.convergenceTolerance): 

389 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

390 converged = True 

391 else: 

392 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

393 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

394 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

395 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

396 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

397 self.log.info("Setting exposure gray photometricity cuts to:") 

398 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

399 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

400 continue 

401 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

402 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

403 

404 cycleNumber += 1 

405 

406 # Log warning if not converged 

407 if not converged: 

408 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber)) 

409 

410 # Do final clean-up iteration 

411 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

412 fgcmFitCycle.fgcmConfig.resetParameters = False 

413 fgcmFitCycle.fgcmConfig.maxIter = 0 

414 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

415 fgcmFitCycle.fgcmConfig.outputStandards = True 

416 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

417 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

418 fgcmFitCycle.initialCycle = False 

419 

420 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

421 fgcmExpInfo, 

422 previousParInfo, 

423 previousParams, 

424 previousSuperStar) 

425 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

426 fgcmStarObservationCat['instMagErr'][obsIndex]) 

427 fgcmFitCycle.setPars(fgcmPars) 

428 fgcmFitCycle.finishSetup() 

429 

430 self.log.info("Running final clean-up fit cycle...") 

431 fgcmFitCycle.run() 

432 

433 self.log.info("Raw repeatability after clean-up cycle is:") 

434 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

435 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

436 continue 

437 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

438 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

439 

440 # Do the outputs. Need to keep track of tract. 

441 

442 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

443 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

444 

445 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

446 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

447 

448 atmSchema = makeAtmSchema() 

449 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

450 

451 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

452 stdSchema = makeStdSchema(len(goodBands)) 

453 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

454 

455 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract, 

456 visitCat, 

457 zptCat, atmCat, stdCat, 

458 self.config.fgcmBuildStars, 

459 self.config.fgcmFitCycle) 

460 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

461 

462 return outStruct