Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for running fgcmcal on a single tract using src tables 

22or sourceTable_visit tables. 

23""" 

24 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from .fgcmBuildStars import FgcmBuildStarsTask, FgcmBuildStarsConfig 

36from .fgcmFitCycle import FgcmFitCycleConfig 

37from .fgcmOutputProducts import FgcmOutputProductsTask 

38from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

39from .utilities import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags 

40from .utilities import makeZptSchema, makeZptCat 

41from .utilities import makeAtmSchema, makeAtmCat 

42from .utilities import makeStdSchema, makeStdCat 

43 

44import fgcm 

45 

46__all__ = ['FgcmCalibrateTractConfigBase', 'FgcmCalibrateTractBaseTask', 'FgcmCalibrateTractRunner'] 

47 

48 

49class FgcmCalibrateTractConfigBase(pexConfig.Config): 

50 """Config for FgcmCalibrateTract""" 

51 

52 fgcmBuildStars = pexConfig.ConfigurableField( 

53 target=FgcmBuildStarsTask, 

54 doc="Task to load and match stars for fgcm", 

55 ) 

56 fgcmFitCycle = pexConfig.ConfigField( 

57 dtype=FgcmFitCycleConfig, 

58 doc="Config to run a single fgcm fit cycle", 

59 ) 

60 fgcmOutputProducts = pexConfig.ConfigurableField( 

61 target=FgcmOutputProductsTask, 

62 doc="Task to output fgcm products", 

63 ) 

64 convergenceTolerance = pexConfig.Field( 

65 doc="Tolerance on repeatability convergence (per band)", 

66 dtype=float, 

67 default=0.005, 

68 ) 

69 maxFitCycles = pexConfig.Field( 

70 doc="Maximum number of fit cycles", 

71 dtype=int, 

72 default=5, 

73 ) 

74 doDebuggingPlots = pexConfig.Field( 

75 doc="Make plots for debugging purposes?", 

76 dtype=bool, 

77 default=False, 

78 ) 

79 

80 def setDefaults(self): 

81 pexConfig.Config.setDefaults(self) 

82 

83 self.fgcmFitCycle.quietMode = True 

84 self.fgcmOutputProducts.doReferenceCalibration = False 

85 self.fgcmOutputProducts.doRefcatOutput = False 

86 self.fgcmOutputProducts.cycleNumber = 0 

87 self.fgcmOutputProducts.photoCal.applyColorTerms = False 

88 

89 def validate(self): 

90 super().validate() 

91 

92 for band in self.fgcmFitCycle.bands: 

93 if not self.fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]: 

94 msg = 'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 

95 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

96 self, msg) 

97 

98 

99class FgcmCalibrateTractRunner(pipeBase.ButlerInitializedTaskRunner): 

100 """Subclass of TaskRunner for FgcmCalibrateTractTask 

101 

102 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 

103 the butler (for persistence and mapper data), and a list of dataRefs 

104 extracted from the command line. This task runs on a constrained set 

105 of dataRefs, typically a single tract. 

106 This class transforms the process arguments generated by the ArgumentParser 

107 into the arguments expected by FgcmCalibrateTractTask.run(). 

108 This runner does not use any parallelization. 

109 """ 

110 

111 @staticmethod 

112 def getTargetList(parsedCmd): 

113 """ 

114 Return a list with one element: a tuple with the butler and 

115 list of dataRefs. 

116 """ 

117 return [(parsedCmd.butler, parsedCmd.id.refList)] 

118 

119 def __call__(self, args): 

120 """ 

121 Parameters 

122 ---------- 

123 args: `tuple` with (butler, dataRefList) 

124 

125 Returns 

126 ------- 

127 exitStatus: `list` with `lsst.pipe.base.Struct` 

128 exitStatus (0: success; 1: failure) 

129 May also contain results if `self.doReturnResults` is `True`. 

130 """ 

131 butler, dataRefList = args 

132 

133 task = self.TaskClass(config=self.config, log=self.log) 

134 

135 exitStatus = 0 

136 if self.doRaise: 

137 results = task.runDataRef(butler, dataRefList) 

138 else: 

139 try: 

140 results = task.runDataRef(butler, dataRefList) 

141 except Exception as e: 

142 exitStatus = 1 

143 task.log.fatal("Failed: %s" % e) 

144 if not isinstance(e, pipeBase.TaskError): 

145 traceback.print_exc(file=sys.stderr) 

146 

147 task.writeMetadata(butler) 

148 

149 if self.doReturnResults: 

150 return [pipeBase.Struct(exitStatus=exitStatus, 

151 results=results)] 

152 else: 

153 return [pipeBase.Struct(exitStatus=exitStatus)] 

154 

155 def run(self, parsedCmd): 

156 """ 

157 Run the task, with no multiprocessing 

158 

159 Parameters 

160 ---------- 

161 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

162 """ 

163 

164 resultList = [] 

165 

166 if self.precall(parsedCmd): 

167 targetList = self.getTargetList(parsedCmd) 

168 resultList = self(targetList[0]) 

169 

170 return resultList 

171 

172 

173class FgcmCalibrateTractBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

174 """Base class to calibrate a single tract using fgcmcal 

175 """ 

176 def __init__(self, butler=None, **kwargs): 

177 """ 

178 Instantiate an `FgcmCalibrateTractTask`. 

179 

180 Parameters 

181 ---------- 

182 butler : `lsst.daf.persistence.Butler`, optional 

183 """ 

184 super().__init__(**kwargs) 

185 self.makeSubtask("fgcmBuildStars", butler=butler) 

186 self.makeSubtask("fgcmOutputProducts", butler=butler) 

187 

188 # no saving of metadata for now 

189 def _getMetadataName(self): 

190 return None 

191 

192 @pipeBase.timeMethod 

193 def runDataRef(self, butler, dataRefs): 

194 """ 

195 Run full FGCM calibration on a single tract, including building star list, 

196 fitting multiple cycles, and making outputs. 

197 

198 Parameters 

199 ---------- 

200 butler: `lsst.daf.persistence.Butler` 

201 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

202 Data references for the input visits. 

203 These may be either per-ccd "src" or per-visit"sourceTable_visit" 

204 references. 

205 

206 Raises 

207 ------ 

208 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 

209 not True, or if fgcmLookUpTable is not available, or if 

210 doSubtractLocalBackground is True and aperture radius cannot be 

211 determined. 

212 """ 

213 datasetType = dataRefs[0].butlerSubset.datasetType 

214 self.log.info("Running with %d %s dataRefs" % (len(dataRefs), datasetType)) 

215 

216 if not butler.datasetExists('fgcmLookUpTable'): 

217 raise RuntimeError("Must run FgcmCalibrateTract with an fgcmLookUpTable") 

218 

219 if not self.config.fgcmBuildStars.doReferenceMatches: 

220 raise RuntimeError("Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches") 

221 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig): 

222 if self.config.fgcmBuildStars.checkAllCcds: 

223 raise RuntimeError("Cannot run FgcmCalibrateTract with " 

224 "fgcmBuildStars.checkAllCcds set to True") 

225 

226 tract = int(dataRefs[0].dataId['tract']) 

227 camera = butler.get('camera') 

228 

229 dataRefDict = {} 

230 dataRefDict['camera'] = camera 

231 dataRefDict['source_catalogs'] = dataRefs 

232 dataRefDict['sourceSchema'] = butler.dataRef('src_schema') 

233 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

234 

235 struct = self.run(dataRefDict, tract, butler=butler, returnCatalogs=False) 

236 

237 visitDataRefName = self.config.fgcmBuildStars.visitDataRefName 

238 ccdDataRefName = self.config.fgcmBuildStars.ccdDataRefName 

239 

240 if struct.photoCalibs is not None: 

241 self.log.info("Outputting photoCalib files.") 

242 

243 filterMapping = {} 

244 for visit, detector, filtername, photoCalib in struct.photoCalibs: 

245 if filtername not in filterMapping: 

246 # We need to find the mapping from encoded filter to dataid filter, 

247 # and this trick allows us to do that. 

248 dataId = {visitDataRefName: visit, 

249 ccdDataRefName: detector} 

250 dataRef = butler.dataRef('raw', dataId=dataId) 

251 filterMapping[filtername] = dataRef.dataId['filter'] 

252 

253 butler.put(photoCalib, 'fgcm_tract_photoCalib', 

254 dataId={visitDataRefName: visit, 

255 ccdDataRefName: detector, 

256 'filter': filterMapping[filtername], 

257 'tract': tract}) 

258 

259 self.log.info("Done outputting photoCalib files.") 

260 

261 if struct.atmospheres is not None: 

262 self.log.info("Outputting atmosphere files.") 

263 for visit, atm in struct.atmospheres: 

264 butler.put(atm, "transmission_atmosphere_fgcm_tract", 

265 dataId={visitDataRefName: visit, 

266 'tract': tract}) 

267 self.log.info("Done outputting atmosphere transmissions.") 

268 

269 return pipeBase.Struct(repeatability=struct.repeatability) 

270 

271 def run(self, dataRefDict, tract, 

272 buildStarsRefObjLoader=None, returnCatalogs=True, butler=None): 

273 """Run the calibrations for a single tract with fgcm. 

274 

275 Parameters 

276 ---------- 

277 dataRefDict : `dict` 

278 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

279 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

280 dataRef dictionary with the following keys. Note that all 

281 keys need not be set based on config parameters. 

282 

283 ``"camera"`` 

284 Camera object (`lsst.afw.cameraGeom.Camera`) 

285 ``"source_catalogs"`` 

286 `list` of dataRefs for input source catalogs. 

287 ``"sourceSchema"`` 

288 Schema for the source catalogs. 

289 ``"fgcmLookUpTable"`` 

290 dataRef for the FGCM look-up table. 

291 ``"calexps"`` 

292 `list` of dataRefs for the input calexps (Gen3 only) 

293 ``"fgcmPhotoCalibs"`` 

294 `dict` of output photoCalib dataRefs. Key is 

295 (tract, visit, detector). (Gen3 only) 

296 Present if doZeropointOutput is True. 

297 ``"fgcmTransmissionAtmospheres"`` 

298 `dict` of output atmosphere transmission dataRefs. 

299 Key is (tract, visit). (Gen3 only) 

300 Present if doAtmosphereOutput is True. 

301 tract : `int` 

302 Tract number 

303 buildStarsRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional 

304 Reference object loader object for fgcmBuildStars. 

305 returnCatalogs : `bool`, optional 

306 Return photoCalibs as per-visit exposure catalogs. 

307 butler : `lsst.daf.persistence.Butler`, optional 

308 Gen2 butler used for reference star outputs 

309 

310 Returns 

311 ------- 

312 outstruct : `lsst.pipe.base.Struct` 

313 Output structure with keys: 

314 

315 offsets : `np.ndarray` 

316 Final reference offsets, per band. 

317 repeatability : `np.ndarray` 

318 Raw fgcm repeatability for bright stars, per band. 

319 atmospheres : `generator` [(`int`, `lsst.afw.image.TransmissionCurve`)] 

320 Generator that returns (visit, transmissionCurve) tuples. 

321 photoCalibs : `generator` [(`int`, `int`, `str`, `lsst.afw.image.PhotoCalib`)] 

322 Generator that returns (visit, ccd, filtername, photoCalib) tuples. 

323 (returned if returnCatalogs is False). 

324 photoCalibCatalogs : `generator` [(`int`, `lsst.afw.table.ExposureCatalog`)] 

325 Generator that returns (visit, exposureCatalog) tuples. 

326 (returned if returnCatalogs is True). 

327 """ 

328 self.log.info("Running on tract %d", (tract)) 

329 

330 # Compute the aperture radius if necessary. This is useful to do now before 

331 # any heavy lifting has happened (fail early). 

332 calibFluxApertureRadius = None 

333 if self.config.fgcmBuildStars.doSubtractLocalBackground: 

334 try: 

335 field = self.config.fgcmBuildStars.instFluxField 

336 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefDict['source_catalogs'][0], 

337 field) 

338 except RuntimeError: 

339 raise RuntimeError("Could not determine aperture radius from %s. " 

340 "Cannot use doSubtractLocalBackground." % 

341 (field)) 

342 

343 # Run the build stars tasks 

344 

345 # Note that we will need visitCat at the end of the procedure for the outputs 

346 if isinstance(butler, dafPersist.Butler): 

347 # Gen2 

348 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict['camera'], 

349 dataRefDict['source_catalogs'], 

350 butler=butler) 

351 else: 

352 # Gen3 

353 cdrd = dataRefDict['calexps'] 

354 groupedDataRefs = self.fgcmBuildStars._findAndGroupDataRefs(dataRefDict['camera'], 

355 dataRefDict['source_catalogs'], 

356 calexpDataRefDict=cdrd) 

357 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(dataRefDict['camera'], groupedDataRefs) 

358 rad = calibFluxApertureRadius 

359 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs, 

360 visitCat, 

361 dataRefDict['sourceSchema'], 

362 dataRefDict['camera'], 

363 calibFluxApertureRadius=rad) 

364 

365 if self.fgcmBuildStars.config.doReferenceMatches: 

366 lutDataRef = dataRefDict['fgcmLookUpTable'] 

367 if buildStarsRefObjLoader is not None: 

368 self.fgcmBuildStars.makeSubtask("fgcmLoadReferenceCatalog", 

369 refObjLoader=buildStarsRefObjLoader) 

370 else: 

371 self.fgcmBuildStars.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

372 else: 

373 lutDataRef = None 

374 

375 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \ 

376 self.fgcmBuildStars.fgcmMatchStars(visitCat, 

377 fgcmStarObservationCat, 

378 lutDataRef=lutDataRef) 

379 

380 # Load the LUT 

381 lutCat = dataRefDict['fgcmLookUpTable'].get() 

382 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

383 dict(self.config.fgcmFitCycle.filterMap)) 

384 del lutCat 

385 

386 # Translate the visit catalog into fgcm format 

387 fgcmExpInfo = translateVisitCatalog(visitCat) 

388 

389 configDict = makeConfigDict(self.config.fgcmFitCycle, self.log, dataRefDict['camera'], 

390 self.config.fgcmFitCycle.maxIterBeforeFinalCycle, 

391 True, False, tract=tract) 

392 # Turn off plotting in tract mode 

393 configDict['doPlots'] = False 

394 

395 # Use the first orientation. 

396 # TODO: DM-21215 will generalize to arbitrary camera orientations 

397 ccdOffsets = computeCcdOffsets(dataRefDict['camera'], fgcmExpInfo['TELROT'][0]) 

398 

399 # Set up the fit cycle task 

400 

401 noFitsDict = {'lutIndex': lutIndexVals, 

402 'lutStd': lutStd, 

403 'expInfo': fgcmExpInfo, 

404 'ccdOffsets': ccdOffsets} 

405 

406 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

407 noFitsDict=noFitsDict, noOutput=True) 

408 

409 # We determine the conversion from the native units (typically radians) to 

410 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

411 # numpy arrays rather than Angles, which would we approximately 600x slower. 

412 conv = fgcmStarObservationCat[0]['ra'].asDegrees() / float(fgcmStarObservationCat[0]['ra']) 

413 

414 # To load the stars, we need an initial parameter object 

415 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

416 fgcmLut, 

417 fgcmExpInfo) 

418 

419 # Match star observations to visits 

420 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

421 # actually be transferred into fgcm using the indexing below. 

422 

423 obsIndex = fgcmStarIndicesCat['obsIndex'] 

424 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], 

425 fgcmStarObservationCat['visit'][obsIndex]) 

426 

427 refMag, refMagErr = extractReferenceMags(fgcmRefCat, 

428 self.config.fgcmFitCycle.bands, 

429 self.config.fgcmFitCycle.filterMap) 

430 refId = fgcmRefCat['fgcm_id'][:] 

431 

432 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

433 fgcmStars.loadStars(fgcmPars, 

434 fgcmStarObservationCat['visit'][obsIndex], 

435 fgcmStarObservationCat['ccd'][obsIndex], 

436 fgcmStarObservationCat['ra'][obsIndex] * conv, 

437 fgcmStarObservationCat['dec'][obsIndex] * conv, 

438 fgcmStarObservationCat['instMag'][obsIndex], 

439 fgcmStarObservationCat['instMagErr'][obsIndex], 

440 fgcmExpInfo['FILTERNAME'][visitIndex], 

441 fgcmStarIdCat['fgcm_id'][:], 

442 fgcmStarIdCat['ra'][:], 

443 fgcmStarIdCat['dec'][:], 

444 fgcmStarIdCat['obsArrIndex'][:], 

445 fgcmStarIdCat['nObs'][:], 

446 obsX=fgcmStarObservationCat['x'][obsIndex], 

447 obsY=fgcmStarObservationCat['y'][obsIndex], 

448 obsDeltaMagBkg=fgcmStarObservationCat['deltaMagBkg'][obsIndex], 

449 psfCandidate=fgcmStarObservationCat['psf_candidate'][obsIndex], 

450 refID=refId, 

451 refMag=refMag, 

452 refMagErr=refMagErr, 

453 flagID=None, 

454 flagFlag=None, 

455 computeNobs=True) 

456 

457 # Clear out some memory 

458 del fgcmStarIdCat 

459 del fgcmStarIndicesCat 

460 del fgcmRefCat 

461 

462 fgcmFitCycle.setLUT(fgcmLut) 

463 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

464 

465 converged = False 

466 cycleNumber = 0 

467 

468 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0 

469 previousParInfo = None 

470 previousParams = None 

471 previousSuperStar = None 

472 

473 while (not converged and cycleNumber < self.config.maxFitCycles): 

474 

475 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

476 

477 if cycleNumber > 0: 

478 # Use parameters from previous cycle 

479 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

480 fgcmExpInfo, 

481 previousParInfo, 

482 previousParams, 

483 previousSuperStar) 

484 # We need to reset the star magnitudes and errors for the next 

485 # cycle 

486 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

487 fgcmStarObservationCat['instMagErr'][obsIndex]) 

488 fgcmFitCycle.initialCycle = False 

489 

490 fgcmFitCycle.setPars(fgcmPars) 

491 fgcmFitCycle.finishSetup() 

492 

493 fgcmFitCycle.run() 

494 

495 # Grab the parameters for the next cycle 

496 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays() 

497 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy() 

498 

499 self.log.info("Raw repeatability after cycle number %d is:" % (cycleNumber)) 

500 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

501 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

502 continue 

503 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

504 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

505 

506 # Check for convergence 

507 if np.alltrue((previousReservedRawRepeatability 

508 - fgcmFitCycle.fgcmPars.compReservedRawRepeatability) 

509 < self.config.convergenceTolerance): 

510 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber)) 

511 converged = True 

512 else: 

513 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut 

514 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut 

515 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle = False 

516 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

517 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

518 self.log.info("Setting exposure gray photometricity cuts to:") 

519 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

520 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

521 continue 

522 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0 

523 self.log.info(" Band %s, photometricity cut: %.2f mmag" % (band, cut)) 

524 

525 cycleNumber += 1 

526 

527 # Log warning if not converged 

528 if not converged: 

529 self.log.warn("Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber)) 

530 

531 # Do final clean-up iteration 

532 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere = False 

533 fgcmFitCycle.fgcmConfig.resetParameters = False 

534 fgcmFitCycle.fgcmConfig.maxIter = 0 

535 fgcmFitCycle.fgcmConfig.outputZeropoints = True 

536 fgcmFitCycle.fgcmConfig.outputStandards = True 

537 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots 

538 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber) 

539 fgcmFitCycle.initialCycle = False 

540 

541 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

542 fgcmExpInfo, 

543 previousParInfo, 

544 previousParams, 

545 previousSuperStar) 

546 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat['instMag'][obsIndex], 

547 fgcmStarObservationCat['instMagErr'][obsIndex]) 

548 fgcmFitCycle.setPars(fgcmPars) 

549 fgcmFitCycle.finishSetup() 

550 

551 self.log.info("Running final clean-up fit cycle...") 

552 fgcmFitCycle.run() 

553 

554 self.log.info("Raw repeatability after clean-up cycle is:") 

555 for i, band in enumerate(fgcmFitCycle.fgcmPars.bands): 

556 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]: 

557 continue 

558 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0 

559 self.log.info(" Band %s, repeatability: %.2f mmag" % (band, rep)) 

560 

561 # Do the outputs. Need to keep track of tract. 

562 

563 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

564 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

565 

566 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

567 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

568 

569 atmSchema = makeAtmSchema() 

570 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

571 

572 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

573 stdSchema = makeStdSchema(len(goodBands)) 

574 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

575 

576 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(dataRefDict, 

577 tract, 

578 visitCat, 

579 zptCat, atmCat, stdCat, 

580 self.config.fgcmBuildStars, 

581 returnCatalogs=returnCatalogs, 

582 butler=butler) 

583 

584 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability 

585 

586 return outStruct