1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Run fgcmcal on a single tract. 

24 

25""" 

26 

27import sys 

28import traceback 

29 

30import numpy as np 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.jointcal.dataIds import PerTractCcdDataIdContainer 

35 

36from .fgcmBuildStars import FgcmBuildStarsTask 

37from .fgcmFitCycle import FgcmFitCycleConfig 

38from .fgcmOutputProducts import FgcmOutputProductsTask 

39from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

40from .utilities import computeCcdOffsets, computeApertureRadius, extractReferenceMags 

41from .utilities import makeZptSchema, makeZptCat 

42from .utilities import makeAtmSchema, makeAtmCat 

43from .utilities import makeStdSchema, makeStdCat 

44 

45import fgcm 

46 

47__all__ = ['FgcmCalibrateTractConfig', 'FgcmCalibrateTractTask', 'FgcmCalibrateTractRunner'] 

48 

49 

50class FgcmCalibrateTractConfig(pexConfig.Config): 

51 """Config for FgcmCalibrateTract""" 

52 

53 fgcmBuildStars = pexConfig.ConfigurableField( 

54 target=FgcmBuildStarsTask, 

55 doc="Task to load and match stars for fgcm", 

56 ) 

57 fgcmFitCycle = pexConfig.ConfigField( 

58 dtype=FgcmFitCycleConfig, 

59 doc="Config to run a single fgcm fit cycle", 

60 ) 

61 fgcmOutputProducts = pexConfig.ConfigurableField( 

62 target=FgcmOutputProductsTask, 

63 doc="Task to output fgcm products", 

64 ) 

65 convergenceTolerance = pexConfig.Field( 

66 doc="Tolerance on repeatability convergence (per band)", 

67 dtype=float, 

68 default=0.005, 

69 ) 

70 maxFitCycles = pexConfig.Field( 

71 doc="Maximum number of fit cycles", 

72 dtype=int, 

73 default=5, 

74 ) 

75 doDebuggingPlots = pexConfig.Field( 

76 doc="Make plots for debugging purposes?", 

77 dtype=bool, 

78 default=False, 

79 ) 

80 

81 def setDefaults(self): 

82 pexConfig.Config.setDefaults(self) 

83 

84 self.fgcmBuildStars.checkAllCcds = False 

85 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

86 self.fgcmFitCycle.useRepeatabilityForExpGrayCuts = [True] 

87 self.fgcmFitCycle.quietMode = True 

88 self.fgcmOutputProducts.doReferenceCalibration = False 

89 self.fgcmOutputProducts.doRefcatOutput = False 

90 self.fgcmOutputProducts.cycleNumber = 0 

91 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

92 

93 

94class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner): 

95 """Subclass of TaskRunner for FgcmCalibrateTractTask 

96 

97 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 

98 the butler (for persistence and mapper data), and a list of dataRefs 

99 extracted from the command line. This task runs on a constrained set 

100 of dataRefs, typically a single tract. 

101 This class transforms the process arguments generated by the ArgumentParser 

102 into the arguments expected by FgcmCalibrateTractTask.run(). 

103 This runner does not use any parallelization. 

104 """ 

105 

106 @staticmethod 

107 def getTargetList(parsedCmd): 

108 """ 

109 Return a list with one element: a tuple with the butler and 

110 list of dataRefs 

111 """ 

112 # we want to combine the butler with any (or no!) dataRefs 

113 return [(parsedCmd.butler, parsedCmd.id.refList)] 

114 

115 def __call__(self, args): 

116 """ 

117 Parameters 

118 ---------- 

119 args: `tuple` with (butler, dataRefList) 

120 

121 Returns 

122 ------- 

123 exitStatus: `list` with `lsst.pipe.base.Struct` 

124 exitStatus (0: success; 1: failure) 

125 """ 

126 butler, dataRefList = args 

127 

128 task = self.TaskClass(config=self.config, log=self.log) 

129 

130 exitStatus = 0 

131 if self.doRaise: 

132 results = task.runDataRef(butler, dataRefList) 

133 else: 

134 try: 

135 results = task.runDataRef(butler, dataRefList) 

136 except Exception as e: 

137 exitStatus = 1 

138 task.log.fatal("Failed: %s" % e) 

139 if not isinstance(e, pipeBase.TaskError): 

140 traceback.print_exc(file=sys.stderr) 

141 

142 task.writeMetadata(butler) 

143 

144 if self.doReturnResults: 

145 return [pipeBase.Struct(exitStatus=exitStatus, 

146 results=results)] 

147 else: 

148 return [pipeBase.Struct(exitStatus=exitStatus)] 

149 

150 def run(self, parsedCmd): 

151 """ 

152 Run the task, with no multiprocessing 

153 

154 Parameters 

155 ---------- 

156 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

157 """ 

158 

159 resultList = [] 

160 

161 if self.precall(parsedCmd): 

162 targetList = self.getTargetList(parsedCmd) 

163 resultList = self(targetList[0]) 

164 

165 return resultList 

166 

167 

168class FgcmCalibrateTractTask(pipeBase.CmdLineTask): 

169 """ 

170 Calibrate a single tract using fgcmcal 

171 """ 

172 

173 ConfigClass = FgcmCalibrateTractConfig 

174 RunnerClass = FgcmCalibrateTractRunner 

175 _DefaultName = "fgcmCalibrateTract" 

176 

177 def __init__(self, butler=None, **kwargs): 

178 """ 

179 Instantiate an `FgcmCalibrateTractTask`. 

180 

181 Parameters 

182 ---------- 

183 butler : `lsst.daf.persistence.Butler` 

184 """ 

185 

186 pipeBase.CmdLineTask.__init__(self, **kwargs) 

187 

188 @classmethod 

189 def _makeArgumentParser(cls): 

190 """Create an argument parser""" 

191 

192 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

193 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789", 

194 ContainerClass=PerTractCcdDataIdContainer) 

195 

196 return parser 

197 

198 # no saving of metadata for now 

199 def _getMetadataName(self): 

200 return None 

201 

202 @pipeBase.timeMethod 

203 def runDataRef(self, butler, dataRefs): 

204 """ 

205 Run full FGCM calibration on a single tract, including building star list, 

206 fitting multiple cycles, and making outputs. 

207 

208 Parameters 

209 ---------- 

210 butler: `lsst.daf.persistence.Butler` 

211 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

212 Data references for the input visits. 

213 If this is an empty list, all visits with src catalogs in 

214 the repository are used. 

215 Only one individual dataRef from a visit need be specified 

216 and the code will find the other source catalogs from 

217 each visit. 

218 

219 Raises 

220 ------ 

221 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 

222 not True, or if fgcmLookUpTable is not available, or if 

223 doSubtractLocalBackground is True and aperture radius cannot be 

224 determined. 

225 """ 

226 

227 if not butler.datasetExists('fgcmLookUpTable'): 

228 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable") 

229 

230 if not self.config.fgcmBuildStars.doReferenceMatches: 

231 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches") 

232 if self.config.fgcmBuildStars.checkAllCcds: 

233 raise RuntimeError("Cannot run FgcmCalibrateTract with fgcmBuildStars.checkAllCcds set to True") 

234 

235 self.makeSubtask("fgcmBuildStars", butler=butler) 

236 self.makeSubtask("fgcmOutputProducts", butler=butler) 

237 

238 # Compute the aperture radius if necessary. This is useful to do now before 

239 # any heavy lifting has happened (fail early). 

240 calibFluxApertureRadius = None 

241 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

242 sourceSchema = butler.get('src_schema').schema 

243 try: 

244 calibFluxApertureRadius = computeApertureRadius(sourceSchema, 

245 self.config.fgcmBuildStars.instFluxField) 

246 except (RuntimeError, LookupError): 

247 raise RuntimeError("Could not determine aperture radius from %s. " 

248 "Cannot use doSubtractLocalBackground." % 

249 (self.config.instFluxField)) 

250 

251 # Run the build stars tasks 

252 tract = dataRefs[0].dataId['tract'] 

253 self.log.info("Running on tract %d" % (tract)) 

254 

255 # Note that we will need visitCat at the end of the procedure for the outputs 

256 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs) 

257 camera = butler.get('camera') 

258 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs) 

259 rad = calibFluxApertureRadius 

260 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs, 

261 visitCat, 

262 calibFluxApertureRadius=rad) 

263 

264 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

265 self.fgcmBuildStars.fgcmMatchStars(butler, 

266 visitCat, 

267 fgcmStarObservationCat) 

268 

269 # Load the LUT 

270 lutCat = butler.get('fgcmLookUpTable') 

271 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

272 dict(self.config.fgcmFitCycle.filterMap)) 

273 del lutCat 

274 

275 # Translate the visit catalog into fgcm format 

276 fgcmExpInfo = translateVisitCatalog(visitCat) 

277 

278 camera = butler.get('camera') 

279 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera, 

280 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

281 True, False, tract=tract) 

282 # Turn off plotting in tract mode 

283 configDict['doPlots'] = False 

284 

285 # Use the first orientation. 

286 # TODO: DM-21215 will generalize to arbitrary camera orientations 

287 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

288 del camera 

289 

290 # Set up the fit cycle task 

291 

292 noFitsDict = {'lutIndex': lutIndexVals, 

293 'lutStd': lutStd, 

294 'expInfo': fgcmExpInfo, 

295 'ccdOffsets': ccdOffsets} 

296 

297 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

298 noFitsDict=noFitsDict, noOutput=True) 

299 

300 # We determine the conversion from the native units (typically radians) to 

301 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

302 # numpy arrays rather than Angles, which would we approximately 600x slower. 

303 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

304 

305 # To load the stars, we need an initial parameter object 

306 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

307 fgcmLut, 

308 fgcmExpInfo) 

309 

310 # Match star observations to visits 

311 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

312 # actually be transferred into fgcm using the indexing below. 

313 

314 obsIndex = fgcmStarIndicesCat['obsIndex'] 

315 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

316 fgcmStarObservationCat['visit'][obsIndex]) 

317 

318 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

319 self.config.fgcmFitCycle.bands, 

320 self.config.fgcmFitCycle.filterMap) 

321 refId = fgcmRefCat['fgcm_id'][:] 

322 

323 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

324 fgcmStars.loadStars(fgcmPars, 

325 fgcmStarObservationCat['visit'][obsIndex], 

326 fgcmStarObservationCat['ccd'][obsIndex], 

327 fgcmStarObservationCat['ra'][obsIndex] * conv, 

328 fgcmStarObservationCat['dec'][obsIndex] * conv, 

329 fgcmStarObservationCat['instMag'][obsIndex], 

330 fgcmStarObservationCat['instMagErr'][obsIndex], 

331 fgcmExpInfo['FILTERNAME'][visitIndex], 

332 fgcmStarIdCat['fgcm_id'][:], 

333 fgcmStarIdCat['ra'][:], 

334 fgcmStarIdCat['dec'][:], 

335 fgcmStarIdCat['obsArrIndex'][:], 

336 fgcmStarIdCat['nObs'][:], 

337 obsX=fgcmStarObservationCat['x'][obsIndex], 

338 obsY=fgcmStarObservationCat['y'][obsIndex], 

339 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

340 refID=refId, 

341 refMag=refMag, 

342 refMagErr=refMagErr, 

343 flagID=None, 

344 flagFlag=None, 

345 computeNobs=True) 

346 

347 fgcmFitCycle.setLUT(fgcmLut) 

348 fgcmFitCycle.setStars(fgcmStars) 

349 

350 # Clear out some memory 

351 # del fgcmStarObservationCat 

352 del fgcmStarIdCat 

353 del fgcmStarIndicesCat 

354 del fgcmRefCat 

355 

356 converged = False 

357 cycleNumber = 0 

358 

359 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

360 previousParInfo = None 

361 previousParams = None 

362 previousSuperStar = None 

363 

364 while (not converged and cycleNumber < self.config.maxFitCycles): 

365 

366 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

367 

368 if cycleNumber > 0: 

369 # Use parameters from previous cycle 

370 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

371 fgcmExpInfo, 

372 previousParInfo, 

373 previousParams, 

374 previousSuperStar) 

375 # We need to reset the star magnitudes and errors for the next 

376 # cycle 

377 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

378 fgcmStarObservationCat['instMagErr'][obsIndex]) 

379 fgcmFitCycle.initialCycle = False 

380 

381 fgcmFitCycle.setPars(fgcmPars) 

382 fgcmFitCycle.finishSetup() 

383 

384 fgcmFitCycle.run() 

385 

386 # Grab the parameters for the next cycle 

387 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

388 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

389 

390 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

391 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

392 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

393 continue 

394 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

395 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

396 

397 # Check for convergence 

398 if np.alltrue((previousReservedRawRepeatability - 

399 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) < 

400 self.config.convergenceTolerance): 

401 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

402 converged = True 

403 else: 

404 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

405 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

406 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

407 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

408 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

409 self.log.info("Setting exposure gray photometricity cuts to:") 

410 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

411 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

412 continue 

413 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

414 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

415 

416 cycleNumber += 1 

417 

418 # Log warning if not converged 

419 if not converged: 

420 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber)) 

421 

422 # Do final clean-up iteration 

423 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

424 fgcmFitCycle.fgcmConfig.resetParameters = False 

425 fgcmFitCycle.fgcmConfig.maxIter = 0 

426 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

427 fgcmFitCycle.fgcmConfig.outputStandards = True 

428 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

429 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

430 fgcmFitCycle.initialCycle = False 

431 

432 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

433 fgcmExpInfo, 

434 previousParInfo, 

435 previousParams, 

436 previousSuperStar) 

437 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

438 fgcmStarObservationCat['instMagErr'][obsIndex]) 

439 fgcmFitCycle.setPars(fgcmPars) 

440 fgcmFitCycle.finishSetup() 

441 

442 self.log.info("Running final clean-up fit cycle...") 

443 fgcmFitCycle.run() 

444 

445 self.log.info("Raw repeatability after clean-up cycle is:") 

446 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

447 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

448 continue 

449 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

450 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

451 

452 # Do the outputs. Need to keep track of tract. 

453 

454 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

455 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

456 

457 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

458 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

459 

460 atmSchema = makeAtmSchema() 

461 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

462 

463 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

464 stdSchema = makeStdSchema(len(goodBands)) 

465 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

466 

467 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract, 

468 visitCat, 

469 zptCat, atmCat, stdCat, 

470 self.config.fgcmBuildStars, 

471 self.config.fgcmFitCycle) 

472 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

473 

474 return outStruct