Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import sys 

37import traceback 

38import copy 

39 

40import numpy as np 

41 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44import lsst.afw.table as afwTable 

45 

46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

47from .utilities import extractReferenceMags 

48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat 

49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

50from .sedterms import SedboundarytermDict, SedtermDict 

51 

52import fgcm 

53 

54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner'] 

55 

56 

57class FgcmFitCycleConfig(pexConfig.Config): 

58 """Config for FgcmFitCycle""" 

59 

60 bands = pexConfig.ListField( 

61 doc="Bands to run calibration", 

62 dtype=str, 

63 default=[], 

64 ) 

65 fitFlag = pexConfig.ListField( 

66 doc=("Flag for which bands are directly constrained in the FGCM fit. " 

67 "Bands set to 0 will have the atmosphere constrained from observations " 

68 "in other bands on the same night. Must be same length as config.bands, " 

69 "and matched band-by-band."), 

70 dtype=int, 

71 default=(0,), 

72 optional=True, 

73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

74 "It will be removed after v20. Use fitBands instead."), 

75 ) 

76 fitBands = pexConfig.ListField( 

77 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

78 "the atmosphere constrained from the 'fitBands' on the same night. " 

79 "Must be a subset of `config.bands`"), 

80 dtype=str, 

81 default=[], 

82 ) 

83 requiredFlag = pexConfig.ListField( 

84 doc=("Flag for which bands are required for a star to be considered a calibration " 

85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must " 

86 "be same length as config.bands, and matched band-by-band."), 

87 dtype=int, 

88 default=(0,), 

89 optional=True, 

90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

91 "It will be removed after v20. Use requiredBands instead."), 

92 ) 

93 requiredBands = pexConfig.ListField( 

94 doc=("Bands that are required for a star to be considered a calibration star. " 

95 "Must be a subset of `config.bands`"), 

96 dtype=str, 

97 default=[], 

98 ) 

99 filterMap = pexConfig.DictField( 

100 doc="Mapping from 'filterName' to band.", 

101 keytype=str, 

102 itemtype=str, 

103 default={}, 

104 ) 

105 doReferenceCalibration = pexConfig.Field( 

106 doc="Use reference catalog as additional constraint on calibration", 

107 dtype=bool, 

108 default=True, 

109 ) 

110 refStarSnMin = pexConfig.Field( 

111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

112 dtype=float, 

113 default=50.0, 

114 ) 

115 refStarOutlierNSig = pexConfig.Field( 

116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

118 dtype=float, 

119 default=4.0, 

120 ) 

121 applyRefStarColorCuts = pexConfig.Field( 

122 doc="Apply color cuts to reference stars?", 

123 dtype=bool, 

124 default=True, 

125 ) 

126 nCore = pexConfig.Field( 

127 doc="Number of cores to use", 

128 dtype=int, 

129 default=4, 

130 ) 

131 nStarPerRun = pexConfig.Field( 

132 doc="Number of stars to run in each chunk", 

133 dtype=int, 

134 default=200000, 

135 ) 

136 nExpPerRun = pexConfig.Field( 

137 doc="Number of exposures to run in each chunk", 

138 dtype=int, 

139 default=1000, 

140 ) 

141 reserveFraction = pexConfig.Field( 

142 doc="Fraction of stars to reserve for testing", 

143 dtype=float, 

144 default=0.1, 

145 ) 

146 freezeStdAtmosphere = pexConfig.Field( 

147 doc="Freeze atmosphere parameters to standard (for testing)", 

148 dtype=bool, 

149 default=False, 

150 ) 

151 precomputeSuperStarInitialCycle = pexConfig.Field( 

152 doc="Precompute superstar flat for initial cycle", 

153 dtype=bool, 

154 default=False, 

155 ) 

156 superStarSubCcd = pexConfig.Field( 

157 doc="Compute superstar flat on sub-ccd scale", 

158 dtype=bool, 

159 default=True, 

160 optional=True, 

161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

162 "It will be removed after v20. Use superStarSubCcdDict instead."), 

163 ) 

164 superStarSubCcdDict = pexConfig.DictField( 

165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

166 "Must have one entry per band."), 

167 keytype=str, 

168 itemtype=bool, 

169 default={}, 

170 ) 

171 superStarSubCcdChebyshevOrder = pexConfig.Field( 

172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

173 "Global default is first-order polynomials, and should be overridden " 

174 "on a camera-by-camera basis depending on the ISR."), 

175 dtype=int, 

176 default=1, 

177 ) 

178 superStarSubCcdTriangular = pexConfig.Field( 

179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

180 "suppress high-order cross terms?"), 

181 dtype=bool, 

182 default=False, 

183 ) 

184 superStarSigmaClip = pexConfig.Field( 

185 doc="Number of sigma to clip outliers when selecting for superstar flats", 

186 dtype=float, 

187 default=5.0, 

188 ) 

189 ccdGraySubCcd = pexConfig.Field( 

190 doc="Compute CCD gray terms on sub-ccd scale", 

191 dtype=bool, 

192 default=False, 

193 optional=True, 

194 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

195 "It will be removed after v20. Use ccdGraySubCcdDict instead."), 

196 ) 

197 ccdGraySubCcdDict = pexConfig.DictField( 

198 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

199 "('ccd gray') on a sub-ccd scale."), 

200 keytype=str, 

201 itemtype=bool, 

202 default={}, 

203 ) 

204 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

205 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

206 dtype=int, 

207 default=1, 

208 ) 

209 ccdGraySubCcdTriangular = pexConfig.Field( 

210 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

211 "suppress high-order cross terms?"), 

212 dtype=bool, 

213 default=True, 

214 ) 

215 cycleNumber = pexConfig.Field( 

216 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

217 "and stage of outlier rejection. See cookbook for details."), 

218 dtype=int, 

219 default=None, 

220 ) 

221 isFinalCycle = pexConfig.Field( 

222 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

223 "selection of stars and photometric exposures, and will output zeropoints " 

224 "and standard stars for use in fgcmOutputProducts"), 

225 dtype=bool, 

226 default=False, 

227 ) 

228 maxIterBeforeFinalCycle = pexConfig.Field( 

229 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

230 "will always be 0 in the final cycle for cleanup and final selection."), 

231 dtype=int, 

232 default=50, 

233 ) 

234 utBoundary = pexConfig.Field( 

235 doc="Boundary (in UTC) from day-to-day", 

236 dtype=float, 

237 default=None, 

238 ) 

239 washMjds = pexConfig.ListField( 

240 doc="Mirror wash MJDs", 

241 dtype=float, 

242 default=(0.0,), 

243 ) 

244 epochMjds = pexConfig.ListField( 

245 doc="Epoch boundaries in MJD", 

246 dtype=float, 

247 default=(0.0,), 

248 ) 

249 minObsPerBand = pexConfig.Field( 

250 doc="Minimum good observations per band", 

251 dtype=int, 

252 default=2, 

253 ) 

254 # TODO: When DM-16511 is done, it will be possible to get the 

255 # telescope latitude directly from the camera. 

256 latitude = pexConfig.Field( 

257 doc="Observatory latitude", 

258 dtype=float, 

259 default=None, 

260 ) 

261 brightObsGrayMax = pexConfig.Field( 

262 doc="Maximum gray extinction to be considered bright observation", 

263 dtype=float, 

264 default=0.15, 

265 ) 

266 minStarPerCcd = pexConfig.Field( 

267 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

268 "CCDs with fewer stars will have their calibration estimated from other " 

269 "CCDs in the same visit, with zeropoint error increased accordingly."), 

270 dtype=int, 

271 default=5, 

272 ) 

273 minCcdPerExp = pexConfig.Field( 

274 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

275 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

276 dtype=int, 

277 default=5, 

278 ) 

279 maxCcdGrayErr = pexConfig.Field( 

280 doc="Maximum error on CCD gray offset to be considered photometric", 

281 dtype=float, 

282 default=0.05, 

283 ) 

284 minStarPerExp = pexConfig.Field( 

285 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

286 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

287 dtype=int, 

288 default=600, 

289 ) 

290 minExpPerNight = pexConfig.Field( 

291 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

292 dtype=int, 

293 default=10, 

294 ) 

295 expGrayInitialCut = pexConfig.Field( 

296 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

297 "observations."), 

298 dtype=float, 

299 default=-0.25, 

300 ) 

301 expGrayPhotometricCut = pexConfig.ListField( 

302 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. " 

303 "Must be same length as config.bands, and matched band-by-band."), 

304 dtype=float, 

305 default=(0.0,), 

306 optional=True, 

307 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

308 "It will be removed after v20. Use expGrayPhotometricCutDict instead."), 

309 ) 

310 expGrayPhotometricCutDict = pexConfig.DictField( 

311 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

312 "('gray term') for a visit to be considered photometric. Must have one " 

313 "entry per band. Broad-band filters should be -0.05."), 

314 keytype=str, 

315 itemtype=float, 

316 default={}, 

317 ) 

318 expGrayHighCut = pexConfig.ListField( 

319 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. " 

320 "Must be same length as config.bands, and matched band-by-band."), 

321 dtype=float, 

322 default=(0.0,), 

323 optional=True, 

324 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

325 "It will be removed after v20. Use expGrayHighCutDict instead."), 

326 ) 

327 expGrayHighCutDict = pexConfig.DictField( 

328 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

329 "('gray term') for a visit to be considered photometric. Must have one " 

330 "entry per band. Broad-band filters should be 0.2."), 

331 keytype=str, 

332 itemtype=float, 

333 default={}, 

334 ) 

335 expGrayRecoverCut = pexConfig.Field( 

336 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

337 "Visits with more gray extinction will only get CCD zeropoints if there are " 

338 "sufficient star observations (minStarPerCcd) on that CCD."), 

339 dtype=float, 

340 default=-1.0, 

341 ) 

342 expVarGrayPhotometricCut = pexConfig.Field( 

343 doc="Maximum exposure variance to be considered possibly photometric", 

344 dtype=float, 

345 default=0.0005, 

346 optional=True, 

347 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

348 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."), 

349 ) 

350 expVarGrayPhotometricCutDict = pexConfig.DictField( 

351 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

352 "photometric. Must have one entry per band. Broad-band filters should be " 

353 "0.0005."), 

354 keytype=str, 

355 itemtype=float, 

356 default={}, 

357 ) 

358 expGrayErrRecoverCut = pexConfig.Field( 

359 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

360 "Visits with more gray variance will only get CCD zeropoints if there are " 

361 "sufficient star observations (minStarPerCcd) on that CCD."), 

362 dtype=float, 

363 default=0.05, 

364 ) 

365 aperCorrFitNBins = pexConfig.Field( 

366 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

367 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

368 "used if available."), 

369 dtype=int, 

370 default=10, 

371 ) 

372 aperCorrInputSlopes = pexConfig.ListField( 

373 doc=("Aperture correction input slope parameters. These are used on the first " 

374 "fit iteration, and aperture correction parameters will be updated from " 

375 "the data if config.aperCorrFitNBins > 0. It is recommended to set this" 

376 "when there is insufficient data to fit the parameters (e.g. tract mode). " 

377 "If set, must be same length as config.bands, and matched band-by-band."), 

378 dtype=float, 

379 default=[], 

380 optional=True, 

381 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

382 "It will be removed after v20. Use aperCorrInputSlopeDict instead."), 

383 ) 

384 aperCorrInputSlopeDict = pexConfig.DictField( 

385 doc=("Per-band specification of aperture correction input slope parameters. These " 

386 "are used on the first fit iteration, and aperture correction parameters will " 

387 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

388 "to set this when there is insufficient data to fit the parameters (e.g. " 

389 "tract mode)."), 

390 keytype=str, 

391 itemtype=float, 

392 default={}, 

393 ) 

394 sedFudgeFactors = pexConfig.ListField( 

395 doc=("Fudge factors for computing linear SED from colors. Must be same length as " 

396 "config.bands, and matched band-by-band."), 

397 dtype=float, 

398 default=(0,), 

399 optional=True, 

400 deprecated=("This field has been deprecated and will be removed after v20. " 

401 "Please use sedSlopeTermMap and sedSlopeMap."), 

402 ) 

403 sedboundaryterms = pexConfig.ConfigField( 

404 doc="Mapping from bands to SED boundary term names used is sedterms.", 

405 dtype=SedboundarytermDict, 

406 ) 

407 sedterms = pexConfig.ConfigField( 

408 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

409 dtype=SedtermDict, 

410 ) 

411 sigFgcmMaxErr = pexConfig.Field( 

412 doc="Maximum mag error for fitting sigma_FGCM", 

413 dtype=float, 

414 default=0.01, 

415 ) 

416 sigFgcmMaxEGray = pexConfig.ListField( 

417 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. " 

418 "May be 1 element (same for all bands) or the same length as config.bands."), 

419 dtype=float, 

420 default=(0.05,), 

421 optional=True, 

422 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

423 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."), 

424 ) 

425 sigFgcmMaxEGrayDict = pexConfig.DictField( 

426 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

427 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

428 "should be 0.05."), 

429 keytype=str, 

430 itemtype=float, 

431 default={}, 

432 ) 

433 ccdGrayMaxStarErr = pexConfig.Field( 

434 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

435 "computation"), 

436 dtype=float, 

437 default=0.10, 

438 ) 

439 approxThroughput = pexConfig.ListField( 

440 doc=("Approximate overall throughput at start of calibration observations. " 

441 "May be 1 element (same for all bands) or the same length as config.bands."), 

442 dtype=float, 

443 default=(1.0, ), 

444 optional=True, 

445 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

446 "It will be removed after v20. Use approxThroughputDict instead."), 

447 ) 

448 approxThroughputDict = pexConfig.DictField( 

449 doc=("Per-band specification of the approximate overall throughput at the start of " 

450 "calibration observations. Must have one entry per band. Typically should " 

451 "be 1.0."), 

452 keytype=str, 

453 itemtype=float, 

454 default={}, 

455 ) 

456 sigmaCalRange = pexConfig.ListField( 

457 doc="Allowed range for systematic error floor estimation", 

458 dtype=float, 

459 default=(0.001, 0.003), 

460 ) 

461 sigmaCalFitPercentile = pexConfig.ListField( 

462 doc="Magnitude percentile range to fit systematic error floor", 

463 dtype=float, 

464 default=(0.05, 0.15), 

465 ) 

466 sigmaCalPlotPercentile = pexConfig.ListField( 

467 doc="Magnitude percentile range to plot systematic error floor", 

468 dtype=float, 

469 default=(0.05, 0.95), 

470 ) 

471 sigma0Phot = pexConfig.Field( 

472 doc="Systematic error floor for all zeropoints", 

473 dtype=float, 

474 default=0.003, 

475 ) 

476 mapLongitudeRef = pexConfig.Field( 

477 doc="Reference longitude for plotting maps", 

478 dtype=float, 

479 default=0.0, 

480 ) 

481 mapNSide = pexConfig.Field( 

482 doc="Healpix nside for plotting maps", 

483 dtype=int, 

484 default=256, 

485 ) 

486 outfileBase = pexConfig.Field( 

487 doc="Filename start for plot output files", 

488 dtype=str, 

489 default=None, 

490 ) 

491 starColorCuts = pexConfig.ListField( 

492 doc="Encoded star-color cuts (to be cleaned up)", 

493 dtype=str, 

494 default=("NO_DATA",), 

495 ) 

496 colorSplitIndices = pexConfig.ListField( 

497 doc="Band indices to use to split stars by color", 

498 dtype=int, 

499 default=None, 

500 optional=True, 

501 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

502 "It will be removed after v20. Use colorSplitBands instead."), 

503 ) 

504 colorSplitBands = pexConfig.ListField( 

505 doc="Band names to use to split stars by color. Must have 2 entries.", 

506 dtype=str, 

507 length=2, 

508 default=('g', 'i'), 

509 ) 

510 modelMagErrors = pexConfig.Field( 

511 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

512 dtype=bool, 

513 default=True, 

514 ) 

515 useQuadraticPwv = pexConfig.Field( 

516 doc="Model PWV with a quadratic term for variation through the night?", 

517 dtype=bool, 

518 default=False, 

519 ) 

520 instrumentParsPerBand = pexConfig.Field( 

521 doc=("Model instrumental parameters per band? " 

522 "Otherwise, instrumental parameters (QE changes with time) are " 

523 "shared among all bands."), 

524 dtype=bool, 

525 default=False, 

526 ) 

527 instrumentSlopeMinDeltaT = pexConfig.Field( 

528 doc=("Minimum time change (in days) between observations to use in constraining " 

529 "instrument slope."), 

530 dtype=float, 

531 default=20.0, 

532 ) 

533 fitMirrorChromaticity = pexConfig.Field( 

534 doc="Fit (intraband) mirror chromatic term?", 

535 dtype=bool, 

536 default=False, 

537 ) 

538 coatingMjds = pexConfig.ListField( 

539 doc="Mirror coating dates in MJD", 

540 dtype=float, 

541 default=(0.0,), 

542 ) 

543 outputStandardsBeforeFinalCycle = pexConfig.Field( 

544 doc="Output standard stars prior to final cycle? Used in debugging.", 

545 dtype=bool, 

546 default=False, 

547 ) 

548 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

549 doc="Output standard stars prior to final cycle? Used in debugging.", 

550 dtype=bool, 

551 default=False, 

552 ) 

553 useRepeatabilityForExpGrayCuts = pexConfig.ListField( 

554 doc=("Use star repeatability (instead of exposures) for computing photometric " 

555 "cuts? Recommended for tract mode or bands with few exposures. " 

556 "May be 1 element (same for all bands) or the same length as config.bands."), 

557 dtype=bool, 

558 default=(False,), 

559 optional=True, 

560 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

561 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."), 

562 ) 

563 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

564 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

565 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

566 keytype=str, 

567 itemtype=bool, 

568 default={}, 

569 ) 

570 autoPhotometricCutNSig = pexConfig.Field( 

571 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

572 "Cut is based on exposure gray width (per band), unless " 

573 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

574 "repeatability is used (also per band)."), 

575 dtype=float, 

576 default=3.0, 

577 ) 

578 autoHighCutNSig = pexConfig.Field( 

579 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

580 "Cut is based on exposure gray width (per band), unless " 

581 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

582 "repeatability is used (also per band)."), 

583 dtype=float, 

584 default=4.0, 

585 ) 

586 quietMode = pexConfig.Field( 

587 doc="Be less verbose with logging.", 

588 dtype=bool, 

589 default=False, 

590 ) 

591 

592 def setDefaults(self): 

593 pass 

594 

595 def validate(self): 

596 super().validate() 

597 

598 for band in self.fitBands: 

599 if band not in self.bands: 

600 msg = 'fitBand %s not in bands' % (band) 

601 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

602 for band in self.requiredBands: 

603 if band not in self.bands: 

604 msg = 'requiredBand %s not in bands' % (band) 

605 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

606 for band in self.colorSplitBands: 

607 if band not in self.bands: 

608 msg = 'colorSplitBand %s not in bands' % (band) 

609 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

610 for band in self.bands: 

611 if band not in self.superStarSubCcdDict: 

612 msg = 'band %s not in superStarSubCcdDict' % (band) 

613 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

614 self, msg) 

615 if band not in self.ccdGraySubCcdDict: 

616 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

617 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

618 self, msg) 

619 if band not in self.expGrayPhotometricCutDict: 

620 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

621 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

622 self, msg) 

623 if band not in self.expGrayHighCutDict: 

624 msg = 'band %s not in expGrayHighCutDict' % (band) 

625 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

626 self, msg) 

627 if band not in self.expVarGrayPhotometricCutDict: 

628 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

629 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

630 self, msg) 

631 if band not in self.sigFgcmMaxEGrayDict: 

632 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

633 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

634 self, msg) 

635 if band not in self.approxThroughputDict: 

636 msg = 'band %s not in approxThroughputDict' % (band) 

637 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

638 self, msg) 

639 if band not in self.useRepeatabilityForExpGrayCutsDict: 

640 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

641 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

642 self, msg) 

643 

644 

645class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner): 

646 """Subclass of TaskRunner for fgcmFitCycleTask 

647 

648 fgcmFitCycleTask.run() takes one argument, the butler, and uses 

649 stars and visits previously extracted from dataRefs by 

650 fgcmBuildStars. 

651 This Runner does not perform any dataRef parallelization, but the FGCM 

652 code called by the Task uses python multiprocessing (see the "ncores" 

653 config option). 

654 """ 

655 

656 @staticmethod 

657 def getTargetList(parsedCmd): 

658 """ 

659 Return a list with one element, the butler. 

660 """ 

661 return [parsedCmd.butler] 

662 

663 def __call__(self, butler): 

664 """ 

665 Parameters 

666 ---------- 

667 butler: `lsst.daf.persistence.Butler` 

668 

669 Returns 

670 ------- 

671 exitStatus: `list` with `pipeBase.Struct` 

672 exitStatus (0: success; 1: failure) 

673 """ 

674 

675 task = self.TaskClass(config=self.config, log=self.log) 

676 

677 exitStatus = 0 

678 if self.doRaise: 

679 task.runDataRef(butler) 

680 else: 

681 try: 

682 task.runDataRef(butler) 

683 except Exception as e: 

684 exitStatus = 1 

685 task.log.fatal("Failed: %s" % e) 

686 if not isinstance(e, pipeBase.TaskError): 

687 traceback.print_exc(file=sys.stderr) 

688 

689 task.writeMetadata(butler) 

690 

691 # The task does not return any results: 

692 return [pipeBase.Struct(exitStatus=exitStatus)] 

693 

694 def run(self, parsedCmd): 

695 """ 

696 Run the task, with no multiprocessing 

697 

698 Parameters 

699 ---------- 

700 parsedCmd: ArgumentParser parsed command line 

701 """ 

702 

703 resultList = [] 

704 

705 if self.precall(parsedCmd): 

706 targetList = self.getTargetList(parsedCmd) 

707 # make sure that we only get 1 

708 resultList = self(targetList[0]) 

709 

710 return resultList 

711 

712 

713class FgcmFitCycleTask(pipeBase.CmdLineTask): 

714 """ 

715 Run Single fit cycle for FGCM global calibration 

716 """ 

717 

718 ConfigClass = FgcmFitCycleConfig 

719 RunnerClass = FgcmFitCycleRunner 

720 _DefaultName = "fgcmFitCycle" 

721 

722 def __init__(self, butler=None, **kwargs): 

723 """ 

724 Instantiate an fgcmFitCycle. 

725 

726 Parameters 

727 ---------- 

728 butler : `lsst.daf.persistence.Butler` 

729 """ 

730 

731 pipeBase.CmdLineTask.__init__(self, **kwargs) 

732 

733 # no saving of metadata for now 

734 def _getMetadataName(self): 

735 return None 

736 

737 @pipeBase.timeMethod 

738 def runDataRef(self, butler): 

739 """ 

740 Run a single fit cycle for FGCM 

741 

742 Parameters 

743 ---------- 

744 butler: `lsst.daf.persistence.Butler` 

745 """ 

746 

747 self._fgcmFitCycle(butler) 

748 

749 def writeConfig(self, butler, clobber=False, doBackup=True): 

750 """Write the configuration used for processing the data, or check that an existing 

751 one is equal to the new one if present. This is an override of the regular 

752 version from pipe_base that knows about fgcmcycle. 

753 

754 Parameters 

755 ---------- 

756 butler : `lsst.daf.persistence.Butler` 

757 Data butler used to write the config. The config is written to dataset type 

758 `CmdLineTask._getConfigName`. 

759 clobber : `bool`, optional 

760 A boolean flag that controls what happens if a config already has been saved: 

761 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 

762 - `False`: raise `TaskError` if this config does not match the existing config. 

763 doBackup : `bool`, optional 

764 Set to `True` to backup the config files if clobbering. 

765 """ 

766 configName = self._getConfigName() 

767 if configName is None: 

768 return 

769 if clobber: 

770 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber) 

771 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber): 

772 # this may be subject to a race condition; see #2789 

773 try: 

774 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber) 

775 except Exception as exc: 

776 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" % 

777 (configName, exc)) 

778 

779 def logConfigMismatch(msg): 

780 self.log.fatal("Comparing configuration: %s", msg) 

781 

782 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

783 raise pipeBase.TaskError( 

784 ("Config does not match existing task config %r on disk; tasks configurations " + 

785 "must be consistent within the same output repo (override with --clobber-config)") % 

786 (configName,)) 

787 else: 

788 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber) 

789 

790 def _fgcmFitCycle(self, butler): 

791 """ 

792 Run the fit cycle 

793 

794 Parameters 

795 ---------- 

796 butler: `lsst.daf.persistence.Butler` 

797 """ 

798 

799 self._checkDatasetsExist(butler) 

800 

801 # Set defaults on whether to output standards and zeropoints 

802 self.maxIter = self.config.maxIterBeforeFinalCycle 

803 self.outputStandards = self.config.outputStandardsBeforeFinalCycle 

804 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle 

805 self.resetFitParameters = True 

806 

807 if self.config.isFinalCycle: 

808 # This is the final fit cycle, so we do not want to reset fit 

809 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

810 # and we always want to output standards and zeropoints 

811 self.maxIter = 0 

812 self.outputStandards = True 

813 self.outputZeropoints = True 

814 self.resetFitParameters = False 

815 

816 camera = butler.get('camera') 

817 configDict = makeConfigDict(self.config, self.log, camera, 

818 self.maxIter, self.resetFitParameters, 

819 self.outputZeropoints) 

820 

821 lutCat = butler.get('fgcmLookUpTable') 

822 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap)) 

823 del lutCat 

824 

825 # next we need the exposure/visit information 

826 

827 # fgcmExpInfo = self._loadVisitCatalog(butler) 

828 visitCat = butler.get('fgcmVisitCatalog') 

829 fgcmExpInfo = translateVisitCatalog(visitCat) 

830 del visitCat 

831 

832 # Use the first orientation. 

833 # TODO: DM-21215 will generalize to arbitrary camera orientations 

834 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

835 

836 noFitsDict = {'lutIndex': lutIndexVals, 

837 'lutStd': lutStd, 

838 'expInfo': fgcmExpInfo, 

839 'ccdOffsets': ccdOffsets} 

840 

841 # set up the fitter object 

842 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

843 noFitsDict=noFitsDict, noOutput=True) 

844 

845 # create the parameter object 

846 if (fgcmFitCycle.initialCycle): 

847 # cycle = 0, initial cycle 

848 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

849 fgcmLut, 

850 fgcmExpInfo) 

851 else: 

852 inParInfo, inParams, inSuperStar = self._loadParameters(butler) 

853 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

854 fgcmExpInfo, 

855 inParInfo, 

856 inParams, 

857 inSuperStar) 

858 

859 lastCycle = configDict['cycleNumber'] - 1 

860 

861 # set up the stars... 

862 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

863 

864 starObs = butler.get('fgcmStarObservations') 

865 starIds = butler.get('fgcmStarIds') 

866 starIndices = butler.get('fgcmStarIndices') 

867 

868 # grab the flagged stars if available 

869 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle): 

870 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle) 

871 flagId = flaggedStars['objId'][:] 

872 flagFlag = flaggedStars['objFlag'][:] 

873 else: 

874 flaggedStars = None 

875 flagId = None 

876 flagFlag = None 

877 

878 if self.config.doReferenceCalibration: 

879 refStars = butler.get('fgcmReferenceStars') 

880 

881 refMag, refMagErr = extractReferenceMags(refStars, 

882 self.config.bands, 

883 self.config.filterMap) 

884 refId = refStars['fgcm_id'][:] 

885 else: 

886 refStars = None 

887 refId = None 

888 refMag = None 

889 refMagErr = None 

890 

891 # match star observations to visits 

892 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

893 # actually be transferred into fgcm using the indexing below. 

894 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

895 

896 # The fgcmStars.loadStars method will copy all the star information into 

897 # special shared memory objects that will not blow up the memory usage when 

898 # used with python multiprocessing. Once all the numbers are copied, 

899 # it is necessary to release all references to the objects that previously 

900 # stored the data to ensure that the garbage collector can clear the memory, 

901 # and ensure that this memory is not copied when multiprocessing kicks in. 

902 

903 # We determine the conversion from the native units (typically radians) to 

904 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

905 # numpy arrays rather than Angles, which would we approximately 600x slower. 

906 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

907 

908 fgcmStars.loadStars(fgcmPars, 

909 starObs['visit'][starIndices['obsIndex']], 

910 starObs['ccd'][starIndices['obsIndex']], 

911 starObs['ra'][starIndices['obsIndex']] * conv, 

912 starObs['dec'][starIndices['obsIndex']] * conv, 

913 starObs['instMag'][starIndices['obsIndex']], 

914 starObs['instMagErr'][starIndices['obsIndex']], 

915 fgcmExpInfo['FILTERNAME'][visitIndex], 

916 starIds['fgcm_id'][:], 

917 starIds['ra'][:], 

918 starIds['dec'][:], 

919 starIds['obsArrIndex'][:], 

920 starIds['nObs'][:], 

921 obsX=starObs['x'][starIndices['obsIndex']], 

922 obsY=starObs['y'][starIndices['obsIndex']], 

923 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

924 refID=refId, 

925 refMag=refMag, 

926 refMagErr=refMagErr, 

927 flagID=flagId, 

928 flagFlag=flagFlag, 

929 computeNobs=True) 

930 

931 # Release all references to temporary objects holding star data (see above) 

932 del starObs 

933 del starIds 

934 del starIndices 

935 del flagId 

936 del flagFlag 

937 del flaggedStars 

938 del refStars 

939 del refId 

940 del refMag 

941 del refMagErr 

942 

943 # and set the bits in the cycle object 

944 fgcmFitCycle.setLUT(fgcmLut) 

945 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

946 fgcmFitCycle.setPars(fgcmPars) 

947 

948 # finish the setup 

949 fgcmFitCycle.finishSetup() 

950 

951 # and run 

952 fgcmFitCycle.run() 

953 

954 ################## 

955 # Persistance 

956 ################## 

957 

958 self._persistFgcmDatasets(butler, fgcmFitCycle) 

959 

960 # Output the config for the next cycle 

961 # We need to make a copy since the input one has been frozen 

962 

963 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

964 i, b in enumerate(self.config.bands)} 

965 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

966 i, band in enumerate(self.config.bands)} 

967 

968 outConfig = copy.copy(self.config) 

969 outConfig.update(cycleNumber=(self.config.cycleNumber + 1), 

970 precomputeSuperStarInitialCycle=False, 

971 freezeStdAtmosphere=False, 

972 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

973 expGrayHighCutDict=updatedHighCutDict) 

974 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

975 outConfig.cycleNumber) 

976 outConfig.save(configFileName) 

977 

978 if self.config.isFinalCycle == 1: 

979 # We are done, ready to output products 

980 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

981 else: 

982 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

983 self.log.info("Be sure to look at:") 

984 self.log.info(" config.expGrayPhotometricCut") 

985 self.log.info(" config.expGrayHighCut") 

986 self.log.info("If you are satisfied with the fit, please set:") 

987 self.log.info(" config.isFinalCycle = True") 

988 

989 def _checkDatasetsExist(self, butler): 

990 """ 

991 Check if necessary datasets exist to run fgcmFitCycle 

992 

993 Parameters 

994 ---------- 

995 butler: `lsst.daf.persistence.Butler` 

996 

997 Raises 

998 ------ 

999 RuntimeError 

1000 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 

1001 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 

1002 If cycleNumber > 0, then also checks for fgcmFitParameters, 

1003 fgcmFlaggedStars. 

1004 """ 

1005 

1006 if not butler.datasetExists('fgcmVisitCatalog'): 

1007 raise RuntimeError("Could not find fgcmVisitCatalog in repo!") 

1008 if not butler.datasetExists('fgcmStarObservations'): 

1009 raise RuntimeError("Could not find fgcmStarObservations in repo!") 

1010 if not butler.datasetExists('fgcmStarIds'): 

1011 raise RuntimeError("Could not find fgcmStarIds in repo!") 

1012 if not butler.datasetExists('fgcmStarIndices'): 

1013 raise RuntimeError("Could not find fgcmStarIndices in repo!") 

1014 if not butler.datasetExists('fgcmLookUpTable'): 

1015 raise RuntimeError("Could not find fgcmLookUpTable in repo!") 

1016 

1017 # Need additional datasets if we are not the initial cycle 

1018 if (self.config.cycleNumber > 0): 

1019 if not butler.datasetExists('fgcmFitParameters', 

1020 fgcmcycle=self.config.cycleNumber-1): 

1021 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" % 

1022 (self.config.cycleNumber-1)) 

1023 if not butler.datasetExists('fgcmFlaggedStars', 

1024 fgcmcycle=self.config.cycleNumber-1): 

1025 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" % 

1026 (self.config.cycleNumber-1)) 

1027 

1028 # And additional dataset if we want reference calibration 

1029 if self.config.doReferenceCalibration: 

1030 if not butler.datasetExists('fgcmReferenceStars'): 

1031 raise RuntimeError("Could not find fgcmReferenceStars in repo, and " 

1032 "doReferenceCalibration is True.") 

1033 

1034 def _loadParameters(self, butler): 

1035 """ 

1036 Load FGCM parameters from a previous fit cycle 

1037 

1038 Parameters 

1039 ---------- 

1040 butler: `lsst.daf.persistence.Butler` 

1041 

1042 Returns 

1043 ------- 

1044 inParInfo: `numpy.ndarray` 

1045 Numpy array parameter information formatted for input to fgcm 

1046 inParameters: `numpy.ndarray` 

1047 Numpy array parameter values formatted for input to fgcm 

1048 inSuperStar: `numpy.array` 

1049 Superstar flat formatted for input to fgcm 

1050 """ 

1051 

1052 # note that we already checked that this is available 

1053 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1) 

1054 

1055 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1056 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1057 

1058 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1059 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1060 (parLutFilterNames.size, )), 

1061 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1062 ('LNTAUUNIT', 'f8'), 

1063 ('LNTAUSLOPEUNIT', 'f8'), 

1064 ('ALPHAUNIT', 'f8'), 

1065 ('LNPWVUNIT', 'f8'), 

1066 ('LNPWVSLOPEUNIT', 'f8'), 

1067 ('LNPWVQUADRATICUNIT', 'f8'), 

1068 ('LNPWVGLOBALUNIT', 'f8'), 

1069 ('O3UNIT', 'f8'), 

1070 ('QESYSUNIT', 'f8'), 

1071 ('FILTEROFFSETUNIT', 'f8'), 

1072 ('HASEXTERNALPWV', 'i2'), 

1073 ('HASEXTERNALTAU', 'i2')]) 

1074 inParInfo['NCCD'] = parCat['nCcd'] 

1075 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1076 inParInfo['FITBANDS'][:] = parFitBands 

1077 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1078 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1079 

1080 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1081 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1082 ('PARLNTAUINTERCEPT', 'f8', 

1083 (parCat['parLnTauIntercept'].size, )), 

1084 ('PARLNTAUSLOPE', 'f8', 

1085 (parCat['parLnTauSlope'].size, )), 

1086 ('PARLNPWVINTERCEPT', 'f8', 

1087 (parCat['parLnPwvIntercept'].size, )), 

1088 ('PARLNPWVSLOPE', 'f8', 

1089 (parCat['parLnPwvSlope'].size, )), 

1090 ('PARLNPWVQUADRATIC', 'f8', 

1091 (parCat['parLnPwvQuadratic'].size, )), 

1092 ('PARQESYSINTERCEPT', 'f8', 

1093 (parCat['parQeSysIntercept'].size, )), 

1094 ('COMPQESYSSLOPE', 'f8', 

1095 (parCat['compQeSysSlope'].size, )), 

1096 ('PARFILTEROFFSET', 'f8', 

1097 (parCat['parFilterOffset'].size, )), 

1098 ('PARFILTEROFFSETFITFLAG', 'i2', 

1099 (parCat['parFilterOffsetFitFlag'].size, )), 

1100 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1101 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1102 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1103 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1104 ('COMPABSTHROUGHPUT', 'f8', 

1105 (parCat['compAbsThroughput'].size, )), 

1106 ('COMPREFOFFSET', 'f8', 

1107 (parCat['compRefOffset'].size, )), 

1108 ('COMPREFSIGMA', 'f8', 

1109 (parCat['compRefSigma'].size, )), 

1110 ('COMPMIRRORCHROMATICITY', 'f8', 

1111 (parCat['compMirrorChromaticity'].size, )), 

1112 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1113 (parCat['mirrorChromaticityPivot'].size, )), 

1114 ('COMPAPERCORRPIVOT', 'f8', 

1115 (parCat['compAperCorrPivot'].size, )), 

1116 ('COMPAPERCORRSLOPE', 'f8', 

1117 (parCat['compAperCorrSlope'].size, )), 

1118 ('COMPAPERCORRSLOPEERR', 'f8', 

1119 (parCat['compAperCorrSlopeErr'].size, )), 

1120 ('COMPAPERCORRRANGE', 'f8', 

1121 (parCat['compAperCorrRange'].size, )), 

1122 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1123 (parCat['compModelErrExptimePivot'].size, )), 

1124 ('COMPMODELERRFWHMPIVOT', 'f8', 

1125 (parCat['compModelErrFwhmPivot'].size, )), 

1126 ('COMPMODELERRSKYPIVOT', 'f8', 

1127 (parCat['compModelErrSkyPivot'].size, )), 

1128 ('COMPMODELERRPARS', 'f8', 

1129 (parCat['compModelErrPars'].size, )), 

1130 ('COMPEXPGRAY', 'f8', 

1131 (parCat['compExpGray'].size, )), 

1132 ('COMPVARGRAY', 'f8', 

1133 (parCat['compVarGray'].size, )), 

1134 ('COMPNGOODSTARPEREXP', 'i4', 

1135 (parCat['compNGoodStarPerExp'].size, )), 

1136 ('COMPSIGFGCM', 'f8', 

1137 (parCat['compSigFgcm'].size, )), 

1138 ('COMPSIGMACAL', 'f8', 

1139 (parCat['compSigmaCal'].size, )), 

1140 ('COMPRETRIEVEDLNPWV', 'f8', 

1141 (parCat['compRetrievedLnPwv'].size, )), 

1142 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1143 (parCat['compRetrievedLnPwvRaw'].size, )), 

1144 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1145 (parCat['compRetrievedLnPwvFlag'].size, )), 

1146 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1147 (parCat['compRetrievedTauNight'].size, ))]) 

1148 

1149 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1150 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1151 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1152 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1153 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1154 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1155 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1156 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1157 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1158 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1159 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1160 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1161 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1162 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1163 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1164 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1165 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1166 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1167 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1168 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1169 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1170 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1171 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1172 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1173 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1174 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1175 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1176 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1177 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1178 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1179 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1180 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1181 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1182 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1183 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1184 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1185 

1186 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1187 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1188 

1189 return (inParInfo, inParams, inSuperStar) 

1190 

1191 def _persistFgcmDatasets(self, butler, fgcmFitCycle): 

1192 """ 

1193 Persist FGCM datasets through the butler. 

1194 

1195 Parameters 

1196 ---------- 

1197 butler: `lsst.daf.persistence.Butler` 

1198 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1199 Fgcm Fit cycle object 

1200 """ 

1201 

1202 # Save the parameters 

1203 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1204 

1205 parSchema = afwTable.Schema() 

1206 

1207 comma = ',' 

1208 lutFilterNameString = comma.join([n.decode('utf-8') 

1209 for n in parInfo['LUTFILTERNAMES'][0]]) 

1210 fitBandString = comma.join([n.decode('utf-8') 

1211 for n in parInfo['FITBANDS'][0]]) 

1212 

1213 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1214 lutFilterNameString, fitBandString) 

1215 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1216 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1217 lutFilterNameString, fitBandString) 

1218 

1219 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber) 

1220 

1221 # Save the indices of the flagged stars 

1222 # (stars that have been (a) reserved from the fit for testing and 

1223 # (b) bad stars that have failed quality checks.) 

1224 flagStarSchema = self._makeFlagStarSchema() 

1225 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1226 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1227 

1228 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber) 

1229 

1230 # Save the zeropoint information and atmospheres only if desired 

1231 if self.outputZeropoints: 

1232 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1233 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1234 

1235 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1236 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1237 

1238 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber) 

1239 

1240 # Save atmosphere values 

1241 # These are generated by the same code that generates zeropoints 

1242 atmSchema = makeAtmSchema() 

1243 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1244 

1245 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber) 

1246 

1247 # Save the standard stars (if configured) 

1248 if self.outputStandards: 

1249 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1250 stdSchema = makeStdSchema(len(goodBands)) 

1251 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1252 

1253 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber) 

1254 

1255 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1256 lutFilterNameString, fitBandString): 

1257 """ 

1258 Make the parameter persistence schema 

1259 

1260 Parameters 

1261 ---------- 

1262 parInfo: `numpy.ndarray` 

1263 Parameter information returned by fgcm 

1264 pars: `numpy.ndarray` 

1265 Parameter values returned by fgcm 

1266 parSuperStarFlat: `numpy.array` 

1267 Superstar flat values returned by fgcm 

1268 lutFilterNameString: `str` 

1269 Combined string of all the lutFilterNames 

1270 fitBandString: `str` 

1271 Combined string of all the fitBands 

1272 

1273 Returns 

1274 ------- 

1275 parSchema: `afwTable.schema` 

1276 """ 

1277 

1278 parSchema = afwTable.Schema() 

1279 

1280 # parameter info section 

1281 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1282 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1283 size=len(lutFilterNameString)) 

1284 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1285 size=len(fitBandString)) 

1286 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1287 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1288 doc='Step units for ln(AOD) slope') 

1289 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1290 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1291 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1292 doc='Step units for ln(pwv) slope') 

1293 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1294 doc='Step units for ln(pwv) quadratic term') 

1295 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1296 doc='Step units for global ln(pwv) parameters') 

1297 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1298 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1299 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1300 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1301 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1302 

1303 # parameter section 

1304 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1305 size=pars['PARALPHA'].size) 

1306 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1307 size=pars['PARO3'].size) 

1308 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1309 doc='ln(Tau) intercept parameter vector', 

1310 size=pars['PARLNTAUINTERCEPT'].size) 

1311 parSchema.addField('parLnTauSlope', type='ArrayD', 

1312 doc='ln(Tau) slope parameter vector', 

1313 size=pars['PARLNTAUSLOPE'].size) 

1314 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1315 size=pars['PARLNPWVINTERCEPT'].size) 

1316 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1317 size=pars['PARLNPWVSLOPE'].size) 

1318 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1319 size=pars['PARLNPWVQUADRATIC'].size) 

1320 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1321 size=pars['PARQESYSINTERCEPT'].size) 

1322 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1323 size=pars[0]['COMPQESYSSLOPE'].size) 

1324 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1325 size=pars['PARFILTEROFFSET'].size) 

1326 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1327 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1328 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1329 doc='Global scale for retrieved ln(pwv)') 

1330 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1331 doc='Global offset for retrieved ln(pwv)') 

1332 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1333 doc='Nightly offset for retrieved ln(pwv)', 

1334 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1335 parSchema.addField('compAbsThroughput', type='ArrayD', 

1336 doc='Absolute throughput (relative to transmission curves)', 

1337 size=pars['COMPABSTHROUGHPUT'].size) 

1338 parSchema.addField('compRefOffset', type='ArrayD', 

1339 doc='Offset between reference stars and calibrated stars', 

1340 size=pars['COMPREFOFFSET'].size) 

1341 parSchema.addField('compRefSigma', type='ArrayD', 

1342 doc='Width of reference star/calibrated star distribution', 

1343 size=pars['COMPREFSIGMA'].size) 

1344 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1345 doc='Computed mirror chromaticity terms', 

1346 size=pars['COMPMIRRORCHROMATICITY'].size) 

1347 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1348 doc='Mirror chromaticity pivot mjd', 

1349 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1350 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1351 size=pars['COMPAPERCORRPIVOT'].size) 

1352 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1353 size=pars['COMPAPERCORRSLOPE'].size) 

1354 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1355 size=pars['COMPAPERCORRSLOPEERR'].size) 

1356 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1357 size=pars['COMPAPERCORRRANGE'].size) 

1358 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1359 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1360 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1361 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1362 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1363 size=pars['COMPMODELERRSKYPIVOT'].size) 

1364 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1365 size=pars['COMPMODELERRPARS'].size) 

1366 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1367 size=pars['COMPEXPGRAY'].size) 

1368 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1369 size=pars['COMPVARGRAY'].size) 

1370 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1371 doc='Computed number of good stars per exposure', 

1372 size=pars['COMPNGOODSTARPEREXP'].size) 

1373 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1374 size=pars['COMPSIGFGCM'].size) 

1375 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1376 size=pars['COMPSIGMACAL'].size) 

1377 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1378 size=pars['COMPRETRIEVEDLNPWV'].size) 

1379 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1380 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1381 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1382 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1383 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1384 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1385 # superstarflat section 

1386 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1387 size=4) 

1388 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1389 size=parSuperStarFlat.size) 

1390 

1391 return parSchema 

1392 

1393 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1394 lutFilterNameString, fitBandString): 

1395 """ 

1396 Make the FGCM parameter catalog for persistence 

1397 

1398 Parameters 

1399 ---------- 

1400 parSchema: `lsst.afw.table.Schema` 

1401 Parameter catalog schema 

1402 pars: `numpy.ndarray` 

1403 FGCM parameters to put into parCat 

1404 parSuperStarFlat: `numpy.array` 

1405 FGCM superstar flat array to put into parCat 

1406 lutFilterNameString: `str` 

1407 Combined string of all the lutFilterNames 

1408 fitBandString: `str` 

1409 Combined string of all the fitBands 

1410 

1411 Returns 

1412 ------- 

1413 parCat: `afwTable.BasicCatalog` 

1414 Atmosphere and instrumental model parameter catalog for persistence 

1415 """ 

1416 

1417 parCat = afwTable.BaseCatalog(parSchema) 

1418 parCat.reserve(1) 

1419 

1420 # The parameter catalog just has one row, with many columns for all the 

1421 # atmosphere and instrument fit parameters 

1422 rec = parCat.addNew() 

1423 

1424 # info section 

1425 rec['nCcd'] = parInfo['NCCD'] 

1426 rec['lutFilterNames'] = lutFilterNameString 

1427 rec['fitBands'] = fitBandString 

1428 # note these are not currently supported here. 

1429 rec['hasExternalPwv'] = 0 

1430 rec['hasExternalTau'] = 0 

1431 

1432 # parameter section 

1433 

1434 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1435 

1436 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1437 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1438 'parQeSysIntercept', 'compQeSysSlope', 

1439 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1440 'parFilterOffset', 'parFilterOffsetFitFlag', 

1441 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1442 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1443 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1444 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1445 'compModelErrSkyPivot', 'compModelErrPars', 

1446 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1447 'compSigmaCal', 

1448 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1449 'compRetrievedTauNight'] 

1450 

1451 for scalarName in scalarNames: 

1452 rec[scalarName] = pars[scalarName.upper()] 

1453 

1454 for arrName in arrNames: 

1455 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1456 

1457 # superstar section 

1458 rec['superstarSize'][:] = parSuperStarFlat.shape 

1459 rec['superstar'][:] = parSuperStarFlat.flatten() 

1460 

1461 return parCat 

1462 

1463 def _makeFlagStarSchema(self): 

1464 """ 

1465 Make the flagged-stars schema 

1466 

1467 Returns 

1468 ------- 

1469 flagStarSchema: `lsst.afw.table.Schema` 

1470 """ 

1471 

1472 flagStarSchema = afwTable.Schema() 

1473 

1474 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1475 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1476 

1477 return flagStarSchema 

1478 

1479 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1480 """ 

1481 Make the flagged star catalog for persistence 

1482 

1483 Parameters 

1484 ---------- 

1485 flagStarSchema: `lsst.afw.table.Schema` 

1486 Flagged star schema 

1487 flagStarStruct: `numpy.ndarray` 

1488 Flagged star structure from fgcm 

1489 

1490 Returns 

1491 ------- 

1492 flagStarCat: `lsst.afw.table.BaseCatalog` 

1493 Flagged star catalog for persistence 

1494 """ 

1495 

1496 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1497 flagStarCat.resize(flagStarStruct.size) 

1498 

1499 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1500 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1501 

1502 return flagStarCat