Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Run fgcmcal on a single tract. 

24 

25""" 

26 

27import sys 

28import traceback 

29 

30import numpy as np 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.jointcal.dataIds import PerTractCcdDataIdContainer 

35 

36from .fgcmBuildStars import FgcmBuildStarsTask 

37from .fgcmFitCycle import FgcmFitCycleConfig 

38from .fgcmOutputProducts import FgcmOutputProductsTask 

39from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

40from .utilities import computeCcdOffsets, computeApertureRadius, extractReferenceMags 

41from .utilities import makeZptSchema, makeZptCat 

42from .utilities import makeAtmSchema, makeAtmCat 

43from .utilities import makeStdSchema, makeStdCat 

44 

45import fgcm 

46 

47__all__ = ['FgcmCalibrateTractConfig', 'FgcmCalibrateTractTask', 'FgcmCalibrateTractRunner'] 

48 

49 

50class FgcmCalibrateTractConfig(pexConfig.Config): 

51 """Config for FgcmCalibrateTract""" 

52 

53 fgcmBuildStars = pexConfig.ConfigurableField( 

54 target=FgcmBuildStarsTask, 

55 doc="Task to load and match stars for fgcm", 

56 ) 

57 fgcmFitCycle = pexConfig.ConfigField( 

58 dtype=FgcmFitCycleConfig, 

59 doc="Config to run a single fgcm fit cycle", 

60 ) 

61 fgcmOutputProducts = pexConfig.ConfigurableField( 

62 target=FgcmOutputProductsTask, 

63 doc="Task to output fgcm products", 

64 ) 

65 convergenceTolerance = pexConfig.Field( 

66 doc="Tolerance on repeatability convergence (per band)", 

67 dtype=float, 

68 default=0.005, 

69 ) 

70 maxFitCycles = pexConfig.Field( 

71 doc="Maximum number of fit cycles", 

72 dtype=int, 

73 default=5, 

74 ) 

75 doDebuggingPlots = pexConfig.Field( 

76 doc="Make plots for debugging purposes?", 

77 dtype=bool, 

78 default=False, 

79 ) 

80 

81 def setDefaults(self): 

82 pexConfig.Config.setDefaults(self) 

83 

84 self.fgcmBuildStars.checkAllCcds = False 

85 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

86 self.fgcmFitCycle.quietMode = True 

87 self.fgcmOutputProducts.doReferenceCalibration = False 

88 self.fgcmOutputProducts.doRefcatOutput = False 

89 self.fgcmOutputProducts.cycleNumber = 0 

90 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

91 

92 def validate(self): 

93 super().validate() 

94 

95 for band in self.fgcmFitCycle.bands: 

96 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]: 

97 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 

98 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

99 self, msg) 

100 

101 

102class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner): 

103 """Subclass of TaskRunner for FgcmCalibrateTractTask 

104 

105 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 

106 the butler (for persistence and mapper data), and a list of dataRefs 

107 extracted from the command line. This task runs on a constrained set 

108 of dataRefs, typically a single tract. 

109 This class transforms the process arguments generated by the ArgumentParser 

110 into the arguments expected by FgcmCalibrateTractTask.run(). 

111 This runner does not use any parallelization. 

112 """ 

113 

114 @staticmethod 

115 def getTargetList(parsedCmd): 

116 """ 

117 Return a list with one element: a tuple with the butler and 

118 list of dataRefs 

119 """ 

120 # we want to combine the butler with any (or no!) dataRefs 

121 return [(parsedCmd.butler, parsedCmd.id.refList)] 

122 

123 def __call__(self, args): 

124 """ 

125 Parameters 

126 ---------- 

127 args: `tuple` with (butler, dataRefList) 

128 

129 Returns 

130 ------- 

131 exitStatus: `list` with `lsst.pipe.base.Struct` 

132 exitStatus (0: success; 1: failure) 

133 """ 

134 butler, dataRefList = args 

135 

136 task = self.TaskClass(config=self.config, log=self.log) 

137 

138 exitStatus = 0 

139 if self.doRaise: 

140 results = task.runDataRef(butler, dataRefList) 

141 else: 

142 try: 

143 results = task.runDataRef(butler, dataRefList) 

144 except Exception as e: 

145 exitStatus = 1 

146 task.log.fatal("Failed: %s" % e) 

147 if not isinstance(e, pipeBase.TaskError): 

148 traceback.print_exc(file=sys.stderr) 

149 

150 task.writeMetadata(butler) 

151 

152 if self.doReturnResults: 

153 return [pipeBase.Struct(exitStatus=exitStatus, 

154 results=results)] 

155 else: 

156 return [pipeBase.Struct(exitStatus=exitStatus)] 

157 

158 def run(self, parsedCmd): 

159 """ 

160 Run the task, with no multiprocessing 

161 

162 Parameters 

163 ---------- 

164 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

165 """ 

166 

167 resultList = [] 

168 

169 if self.precall(parsedCmd): 

170 targetList = self.getTargetList(parsedCmd) 

171 resultList = self(targetList[0]) 

172 

173 return resultList 

174 

175 

176class FgcmCalibrateTractTask(pipeBase.CmdLineTask): 

177 """ 

178 Calibrate a single tract using fgcmcal 

179 """ 

180 

181 ConfigClass = FgcmCalibrateTractConfig 

182 RunnerClass = FgcmCalibrateTractRunner 

183 _DefaultName = "fgcmCalibrateTract" 

184 

185 def __init__(self, butler=None, **kwargs): 

186 """ 

187 Instantiate an `FgcmCalibrateTractTask`. 

188 

189 Parameters 

190 ---------- 

191 butler : `lsst.daf.persistence.Butler` 

192 """ 

193 

194 pipeBase.CmdLineTask.__init__(self, **kwargs) 

195 

196 @classmethod 

197 def _makeArgumentParser(cls): 

198 """Create an argument parser""" 

199 

200 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

201 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789", 

202 ContainerClass=PerTractCcdDataIdContainer) 

203 

204 return parser 

205 

206 # no saving of metadata for now 

207 def _getMetadataName(self): 

208 return None 

209 

210 @pipeBase.timeMethod 

211 def runDataRef(self, butler, dataRefs): 

212 """ 

213 Run full FGCM calibration on a single tract, including building star list, 

214 fitting multiple cycles, and making outputs. 

215 

216 Parameters 

217 ---------- 

218 butler: `lsst.daf.persistence.Butler` 

219 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

220 Data references for the input visits. 

221 If this is an empty list, all visits with src catalogs in 

222 the repository are used. 

223 Only one individual dataRef from a visit need be specified 

224 and the code will find the other source catalogs from 

225 each visit. 

226 

227 Raises 

228 ------ 

229 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 

230 not True, or if fgcmLookUpTable is not available, or if 

231 doSubtractLocalBackground is True and aperture radius cannot be 

232 determined. 

233 """ 

234 

235 if not butler.datasetExists('fgcmLookUpTable'): 

236 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable") 

237 

238 if not self.config.fgcmBuildStars.doReferenceMatches: 

239 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches") 

240 if self.config.fgcmBuildStars.checkAllCcds: 

241 raise RuntimeError("Cannot run FgcmCalibrateTract with fgcmBuildStars.checkAllCcds set to True") 

242 

243 self.makeSubtask("fgcmBuildStars", butler=butler) 

244 self.makeSubtask("fgcmOutputProducts", butler=butler) 

245 

246 # Compute the aperture radius if necessary. This is useful to do now before 

247 # any heavy lifting has happened (fail early). 

248 calibFluxApertureRadius = None 

249 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

250 sourceSchema = butler.get('src_schema').schema 

251 try: 

252 calibFluxApertureRadius = computeApertureRadius(sourceSchema, 

253 self.config.fgcmBuildStars.instFluxField) 

254 except (RuntimeError, LookupError): 

255 raise RuntimeError("Could not determine aperture radius from %s. " 

256 "Cannot use doSubtractLocalBackground." % 

257 (self.config.instFluxField)) 

258 

259 # Run the build stars tasks 

260 tract = dataRefs[0].dataId['tract'] 

261 self.log.info("Running on tract %d" % (tract)) 

262 

263 # Note that we will need visitCat at the end of the procedure for the outputs 

264 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs) 

265 camera = butler.get('camera') 

266 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs) 

267 rad = calibFluxApertureRadius 

268 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs, 

269 visitCat, 

270 calibFluxApertureRadius=rad) 

271 

272 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

273 self.fgcmBuildStars.fgcmMatchStars(butler, 

274 visitCat, 

275 fgcmStarObservationCat) 

276 

277 # Load the LUT 

278 lutCat = butler.get('fgcmLookUpTable') 

279 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

280 dict(self.config.fgcmFitCycle.filterMap)) 

281 del lutCat 

282 

283 # Translate the visit catalog into fgcm format 

284 fgcmExpInfo = translateVisitCatalog(visitCat) 

285 

286 camera = butler.get('camera') 

287 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera, 

288 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

289 True, False, tract=tract) 

290 # Turn off plotting in tract mode 

291 configDict['doPlots'] = False 

292 

293 # Use the first orientation. 

294 # TODO: DM-21215 will generalize to arbitrary camera orientations 

295 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

296 del camera 

297 

298 # Set up the fit cycle task 

299 

300 noFitsDict = {'lutIndex': lutIndexVals, 

301 'lutStd': lutStd, 

302 'expInfo': fgcmExpInfo, 

303 'ccdOffsets': ccdOffsets} 

304 

305 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

306 noFitsDict=noFitsDict, noOutput=True) 

307 

308 # We determine the conversion from the native units (typically radians) to 

309 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

310 # numpy arrays rather than Angles, which would we approximately 600x slower. 

311 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

312 

313 # To load the stars, we need an initial parameter object 

314 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

315 fgcmLut, 

316 fgcmExpInfo) 

317 

318 # Match star observations to visits 

319 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

320 # actually be transferred into fgcm using the indexing below. 

321 

322 obsIndex = fgcmStarIndicesCat['obsIndex'] 

323 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

324 fgcmStarObservationCat['visit'][obsIndex]) 

325 

326 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

327 self.config.fgcmFitCycle.bands, 

328 self.config.fgcmFitCycle.filterMap) 

329 refId = fgcmRefCat['fgcm_id'][:] 

330 

331 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

332 fgcmStars.loadStars(fgcmPars, 

333 fgcmStarObservationCat['visit'][obsIndex], 

334 fgcmStarObservationCat['ccd'][obsIndex], 

335 fgcmStarObservationCat['ra'][obsIndex] * conv, 

336 fgcmStarObservationCat['dec'][obsIndex] * conv, 

337 fgcmStarObservationCat['instMag'][obsIndex], 

338 fgcmStarObservationCat['instMagErr'][obsIndex], 

339 fgcmExpInfo['FILTERNAME'][visitIndex], 

340 fgcmStarIdCat['fgcm_id'][:], 

341 fgcmStarIdCat['ra'][:], 

342 fgcmStarIdCat['dec'][:], 

343 fgcmStarIdCat['obsArrIndex'][:], 

344 fgcmStarIdCat['nObs'][:], 

345 obsX=fgcmStarObservationCat['x'][obsIndex], 

346 obsY=fgcmStarObservationCat['y'][obsIndex], 

347 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

348 refID=refId, 

349 refMag=refMag, 

350 refMagErr=refMagErr, 

351 flagID=None, 

352 flagFlag=None, 

353 computeNobs=True) 

354 

355 # Clear out some memory 

356 del fgcmStarIdCat 

357 del fgcmStarIndicesCat 

358 del fgcmRefCat 

359 

360 fgcmFitCycle.setLUT(fgcmLut) 

361 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

362 

363 converged = False 

364 cycleNumber = 0 

365 

366 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

367 previousParInfo = None 

368 previousParams = None 

369 previousSuperStar = None 

370 

371 while (not converged and cycleNumber < self.config.maxFitCycles): 

372 

373 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

374 

375 if cycleNumber > 0: 

376 # Use parameters from previous cycle 

377 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

378 fgcmExpInfo, 

379 previousParInfo, 

380 previousParams, 

381 previousSuperStar) 

382 # We need to reset the star magnitudes and errors for the next 

383 # cycle 

384 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

385 fgcmStarObservationCat['instMagErr'][obsIndex]) 

386 fgcmFitCycle.initialCycle = False 

387 

388 fgcmFitCycle.setPars(fgcmPars) 

389 fgcmFitCycle.finishSetup() 

390 

391 fgcmFitCycle.run() 

392 

393 # Grab the parameters for the next cycle 

394 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

395 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

396 

397 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

398 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

399 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

400 continue 

401 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

402 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

403 

404 # Check for convergence 

405 if np.alltrue((previousReservedRawRepeatability - 

406 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) < 

407 self.config.convergenceTolerance): 

408 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

409 converged = True 

410 else: 

411 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

412 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

413 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

414 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

415 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

416 self.log.info("Setting exposure gray photometricity cuts to:") 

417 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

418 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

419 continue 

420 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

421 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

422 

423 cycleNumber += 1 

424 

425 # Log warning if not converged 

426 if not converged: 

427 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber)) 

428 

429 # Do final clean-up iteration 

430 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

431 fgcmFitCycle.fgcmConfig.resetParameters = False 

432 fgcmFitCycle.fgcmConfig.maxIter = 0 

433 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

434 fgcmFitCycle.fgcmConfig.outputStandards = True 

435 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

436 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

437 fgcmFitCycle.initialCycle = False 

438 

439 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

440 fgcmExpInfo, 

441 previousParInfo, 

442 previousParams, 

443 previousSuperStar) 

444 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

445 fgcmStarObservationCat['instMagErr'][obsIndex]) 

446 fgcmFitCycle.setPars(fgcmPars) 

447 fgcmFitCycle.finishSetup() 

448 

449 self.log.info("Running final clean-up fit cycle...") 

450 fgcmFitCycle.run() 

451 

452 self.log.info("Raw repeatability after clean-up cycle is:") 

453 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

454 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

455 continue 

456 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

457 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

458 

459 # Do the outputs. Need to keep track of tract. 

460 

461 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

462 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

463 

464 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

465 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

466 

467 atmSchema = makeAtmSchema() 

468 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

469 

470 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

471 stdSchema = makeStdSchema(len(goodBands)) 

472 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

473 

474 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract, 

475 visitCat, 

476 zptCat, atmCat, stdCat, 

477 self.config.fgcmBuildStars, 

478 self.config.fgcmFitCycle) 

479 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

480 

481 return outStruct