1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Run fgcmcal on a single tract. 

24 

25""" 

26 

27import sys 

28import traceback 

29 

30import numpy as np 

31 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.jointcal.dataIds import PerTractCcdDataIdContainer 

35 

36from .fgcmBuildStars import FgcmBuildStarsTask 

37from .fgcmFitCycle import FgcmFitCycleConfig 

38from .fgcmOutputProducts import FgcmOutputProductsTask 

39from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

40from .utilities import computeCcdOffsets, computeApertureRadius, extractReferenceMags 

41from .utilities import makeZptSchema, makeZptCat 

42from .utilities import makeAtmSchema, makeAtmCat 

43from .utilities import makeStdSchema, makeStdCat 

44 

45import fgcm 

46 

47__all__ = ['FgcmCalibrateTractConfig', 'FgcmCalibrateTractTask', 'FgcmCalibrateTractRunner'] 

48 

49 

50class FgcmCalibrateTractConfig(pexConfig.Config): 

51 """Config for FgcmCalibrateTract""" 

52 

53 fgcmBuildStars = pexConfig.ConfigurableField( 

54 target=FgcmBuildStarsTask, 

55 doc="Task to load and match stars for fgcm", 

56 ) 

57 fgcmFitCycle = pexConfig.ConfigField( 

58 dtype=FgcmFitCycleConfig, 

59 doc="Config to run a single fgcm fit cycle", 

60 ) 

61 fgcmOutputProducts = pexConfig.ConfigurableField( 

62 target=FgcmOutputProductsTask, 

63 doc="Task to output fgcm products", 

64 ) 

65 convergenceTolerance = pexConfig.Field( 

66 doc="Tolerance on repeatability convergence (per band)", 

67 dtype=float, 

68 default=0.005, 

69 ) 

70 maxFitCycles = pexConfig.Field( 

71 doc="Maximum number of fit cycles", 

72 dtype=int, 

73 default=5, 

74 ) 

75 doDebuggingPlots = pexConfig.Field( 

76 doc="Make plots for debugging purposes?", 

77 dtype=bool, 

78 default=False, 

79 ) 

80 

81 def setDefaults(self): 

82 pexConfig.Config.setDefaults(self) 

83 

84 self.fgcmBuildStars.checkAllCcds = False 

85 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

86 self.fgcmFitCycle.quietMode = True 

87 self.fgcmOutputProducts.doReferenceCalibration = False 

88 self.fgcmOutputProducts.doRefcatOutput = False 

89 self.fgcmOutputProducts.cycleNumber = 0 

90 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

91 

92 def validate(self): 

93 super().validate() 

94 

95 for band in self.fgcmFitCycle.bands: 

96 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]: 

97 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 

98 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

99 self, msg) 

100 

101 

102class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner): 

103 """Subclass of TaskRunner for FgcmCalibrateTractTask 

104 

105 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 

106 the butler (for persistence and mapper data), and a list of dataRefs 

107 extracted from the command line. This task runs on a constrained set 

108 of dataRefs, typically a single tract. 

109 This class transforms the process arguments generated by the ArgumentParser 

110 into the arguments expected by FgcmCalibrateTractTask.run(). 

111 This runner does not use any parallelization. 

112 """ 

113 

114 @staticmethod 

115 def getTargetList(parsedCmd): 

116 """ 

117 Return a list with one element: a tuple with the butler and 

118 list of dataRefs 

119 """ 

120 # we want to combine the butler with any (or no!) dataRefs 

121 return [(parsedCmd.butler, parsedCmd.id.refList)] 

122 

123 def __call__(self, args): 

124 """ 

125 Parameters 

126 ---------- 

127 args: `tuple` with (butler, dataRefList) 

128 

129 Returns 

130 ------- 

131 exitStatus: `list` with `lsst.pipe.base.Struct` 

132 exitStatus (0: success; 1: failure) 

133 """ 

134 butler, dataRefList = args 

135 

136 task = self.TaskClass(config=self.config, log=self.log) 

137 

138 exitStatus = 0 

139 if self.doRaise: 

140 results = task.runDataRef(butler, dataRefList) 

141 else: 

142 try: 

143 results = task.runDataRef(butler, dataRefList) 

144 except Exception as e: 

145 exitStatus = 1 

146 task.log.fatal("Failed: %s" % e) 

147 if not isinstance(e, pipeBase.TaskError): 

148 traceback.print_exc(file=sys.stderr) 

149 

150 task.writeMetadata(butler) 

151 

152 if self.doReturnResults: 

153 return [pipeBase.Struct(exitStatus=exitStatus, 

154 results=results)] 

155 else: 

156 return [pipeBase.Struct(exitStatus=exitStatus)] 

157 

158 def run(self, parsedCmd): 

159 """ 

160 Run the task, with no multiprocessing 

161 

162 Parameters 

163 ---------- 

164 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

165 """ 

166 

167 resultList = [] 

168 

169 if self.precall(parsedCmd): 

170 targetList = self.getTargetList(parsedCmd) 

171 resultList = self(targetList[0]) 

172 

173 return resultList 

174 

175 

176class FgcmCalibrateTractTask(pipeBase.CmdLineTask): 

177 """ 

178 Calibrate a single tract using fgcmcal 

179 """ 

180 

181 ConfigClass = FgcmCalibrateTractConfig 

182 RunnerClass = FgcmCalibrateTractRunner 

183 _DefaultName = "fgcmCalibrateTract" 

184 

185 def __init__(self, butler=None, **kwargs): 

186 """ 

187 Instantiate an `FgcmCalibrateTractTask`. 

188 

189 Parameters 

190 ---------- 

191 butler : `lsst.daf.persistence.Butler` 

192 """ 

193 

194 pipeBase.CmdLineTask.__init__(self, **kwargs) 

195 

196 @classmethod 

197 def _makeArgumentParser(cls): 

198 """Create an argument parser""" 

199 

200 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

201 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789", 

202 ContainerClass=PerTractCcdDataIdContainer) 

203 

204 return parser 

205 

206 # no saving of metadata for now 

207 def _getMetadataName(self): 

208 return None 

209 

210 @pipeBase.timeMethod 

211 def runDataRef(self, butler, dataRefs): 

212 """ 

213 Run full FGCM calibration on a single tract, including building star list, 

214 fitting multiple cycles, and making outputs. 

215 

216 Parameters 

217 ---------- 

218 butler: `lsst.daf.persistence.Butler` 

219 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

220 Data references for the input visits. 

221 If this is an empty list, all visits with src catalogs in 

222 the repository are used. 

223 Only one individual dataRef from a visit need be specified 

224 and the code will find the other source catalogs from 

225 each visit. 

226 

227 Raises 

228 ------ 

229 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 

230 not True, or if fgcmLookUpTable is not available, or if 

231 doSubtractLocalBackground is True and aperture radius cannot be 

232 determined. 

233 """ 

234 

235 if not butler.datasetExists('fgcmLookUpTable'): 

236 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable") 

237 

238 if not self.config.fgcmBuildStars.doReferenceMatches: 

239 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches") 

240 if self.config.fgcmBuildStars.checkAllCcds: 

241 raise RuntimeError("Cannot run FgcmCalibrateTract with fgcmBuildStars.checkAllCcds set to True") 

242 

243 self.makeSubtask("fgcmBuildStars", butler=butler) 

244 self.makeSubtask("fgcmOutputProducts", butler=butler) 

245 

246 # Compute the aperture radius if necessary. This is useful to do now before 

247 # any heavy lifting has happened (fail early). 

248 calibFluxApertureRadius = None 

249 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

250 sourceSchema = butler.get('src_schema').schema 

251 try: 

252 calibFluxApertureRadius = computeApertureRadius(sourceSchema, 

253 self.config.fgcmBuildStars.instFluxField) 

254 except (RuntimeError, LookupError): 

255 raise RuntimeError("Could not determine aperture radius from %s. " 

256 "Cannot use doSubtractLocalBackground." % 

257 (self.config.instFluxField)) 

258 

259 # Run the build stars tasks 

260 tract = dataRefs[0].dataId['tract'] 

261 self.log.info("Running on tract %d" % (tract)) 

262 

263 # Note that we will need visitCat at the end of the procedure for the outputs 

264 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs) 

265 camera = butler.get('camera') 

266 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs) 

267 rad = calibFluxApertureRadius 

268 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs, 

269 visitCat, 

270 calibFluxApertureRadius=rad) 

271 

272 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

273 self.fgcmBuildStars.fgcmMatchStars(butler, 

274 visitCat, 

275 fgcmStarObservationCat) 

276 

277 # Load the LUT 

278 lutCat = butler.get('fgcmLookUpTable') 

279 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

280 dict(self.config.fgcmFitCycle.filterMap)) 

281 del lutCat 

282 

283 # Translate the visit catalog into fgcm format 

284 fgcmExpInfo = translateVisitCatalog(visitCat) 

285 

286 camera = butler.get('camera') 

287 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, camera, 

288 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

289 True, False, tract=tract) 

290 # Turn off plotting in tract mode 

291 configDict['doPlots'] = False 

292 

293 # Use the first orientation. 

294 # TODO: DM-21215 will generalize to arbitrary camera orientations 

295 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

296 del camera 

297 

298 # Set up the fit cycle task 

299 

300 noFitsDict = {'lutIndex': lutIndexVals, 

301 'lutStd': lutStd, 

302 'expInfo': fgcmExpInfo, 

303 'ccdOffsets': ccdOffsets} 

304 

305 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

306 noFitsDict=noFitsDict, noOutput=True) 

307 

308 # We determine the conversion from the native units (typically radians) to 

309 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

310 # numpy arrays rather than Angles, which would we approximately 600x slower. 

311 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

312 

313 # To load the stars, we need an initial parameter object 

314 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

315 fgcmLut, 

316 fgcmExpInfo) 

317 

318 # Match star observations to visits 

319 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

320 # actually be transferred into fgcm using the indexing below. 

321 

322 obsIndex = fgcmStarIndicesCat['obsIndex'] 

323 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

324 fgcmStarObservationCat['visit'][obsIndex]) 

325 

326 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

327 self.config.fgcmFitCycle.bands, 

328 self.config.fgcmFitCycle.filterMap) 

329 refId = fgcmRefCat['fgcm_id'][:] 

330 

331 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

332 fgcmStars.loadStars(fgcmPars, 

333 fgcmStarObservationCat['visit'][obsIndex], 

334 fgcmStarObservationCat['ccd'][obsIndex], 

335 fgcmStarObservationCat['ra'][obsIndex] * conv, 

336 fgcmStarObservationCat['dec'][obsIndex] * conv, 

337 fgcmStarObservationCat['instMag'][obsIndex], 

338 fgcmStarObservationCat['instMagErr'][obsIndex], 

339 fgcmExpInfo['FILTERNAME'][visitIndex], 

340 fgcmStarIdCat['fgcm_id'][:], 

341 fgcmStarIdCat['ra'][:], 

342 fgcmStarIdCat['dec'][:], 

343 fgcmStarIdCat['obsArrIndex'][:], 

344 fgcmStarIdCat['nObs'][:], 

345 obsX=fgcmStarObservationCat['x'][obsIndex], 

346 obsY=fgcmStarObservationCat['y'][obsIndex], 

347 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

348 refID=refId, 

349 refMag=refMag, 

350 refMagErr=refMagErr, 

351 flagID=None, 

352 flagFlag=None, 

353 computeNobs=True) 

354 

355 fgcmFitCycle.setLUT(fgcmLut) 

356 fgcmFitCycle.setStars(fgcmStars) 

357 

358 # Clear out some memory 

359 # del fgcmStarObservationCat 

360 del fgcmStarIdCat 

361 del fgcmStarIndicesCat 

362 del fgcmRefCat 

363 

364 converged = False 

365 cycleNumber = 0 

366 

367 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

368 previousParInfo = None 

369 previousParams = None 

370 previousSuperStar = None 

371 

372 while (not converged and cycleNumber < self.config.maxFitCycles): 

373 

374 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

375 

376 if cycleNumber > 0: 

377 # Use parameters from previous cycle 

378 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

379 fgcmExpInfo, 

380 previousParInfo, 

381 previousParams, 

382 previousSuperStar) 

383 # We need to reset the star magnitudes and errors for the next 

384 # cycle 

385 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

386 fgcmStarObservationCat['instMagErr'][obsIndex]) 

387 fgcmFitCycle.initialCycle = False 

388 

389 fgcmFitCycle.setPars(fgcmPars) 

390 fgcmFitCycle.finishSetup() 

391 

392 fgcmFitCycle.run() 

393 

394 # Grab the parameters for the next cycle 

395 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

396 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

397 

398 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

399 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

400 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

401 continue 

402 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

403 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

404 

405 # Check for convergence 

406 if np.alltrue((previousReservedRawRepeatability - 

407 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) < 

408 self.config.convergenceTolerance): 

409 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

410 converged = True 

411 else: 

412 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

413 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

414 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

415 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

416 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

417 self.log.info("Setting exposure gray photometricity cuts to:") 

418 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

419 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

420 continue 

421 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

422 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

423 

424 cycleNumber += 1 

425 

426 # Log warning if not converged 

427 if not converged: 

428 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber)) 

429 

430 # Do final clean-up iteration 

431 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

432 fgcmFitCycle.fgcmConfig.resetParameters = False 

433 fgcmFitCycle.fgcmConfig.maxIter = 0 

434 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

435 fgcmFitCycle.fgcmConfig.outputStandards = True 

436 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

437 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

438 fgcmFitCycle.initialCycle = False 

439 

440 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

441 fgcmExpInfo, 

442 previousParInfo, 

443 previousParams, 

444 previousSuperStar) 

445 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

446 fgcmStarObservationCat['instMagErr'][obsIndex]) 

447 fgcmFitCycle.setPars(fgcmPars) 

448 fgcmFitCycle.finishSetup() 

449 

450 self.log.info("Running final clean-up fit cycle...") 

451 fgcmFitCycle.run() 

452 

453 self.log.info("Raw repeatability after clean-up cycle is:") 

454 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

455 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

456 continue 

457 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

458 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

459 

460 # Do the outputs. Need to keep track of tract. 

461 

462 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

463 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

464 

465 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

466 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

467 

468 atmSchema = makeAtmSchema() 

469 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

470 

471 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

472 stdSchema = makeStdSchema(len(goodBands)) 

473 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

474 

475 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract, 

476 visitCat, 

477 zptCat, atmCat, stdCat, 

478 self.config.fgcmBuildStars, 

479 self.config.fgcmFitCycle) 

480 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

481 

482 return outStruct