Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import sys 

37import traceback 

38import copy 

39 

40import numpy as np 

41 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44import lsst.afw.table as afwTable 

45 

46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

47from .utilities import extractReferenceMags 

48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat 

49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

50from .sedterms import SedboundarytermDict, SedtermDict 

51 

52import fgcm 

53 

54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner'] 

55 

56 

57class FgcmFitCycleConfig(pexConfig.Config): 

58 """Config for FgcmFitCycle""" 

59 

60 bands = pexConfig.ListField( 

61 doc="Bands to run calibration", 

62 dtype=str, 

63 default=[], 

64 ) 

65 fitFlag = pexConfig.ListField( 

66 doc=("Flag for which bands are directly constrained in the FGCM fit. " 

67 "Bands set to 0 will have the atmosphere constrained from observations " 

68 "in other bands on the same night. Must be same length as config.bands, " 

69 "and matched band-by-band."), 

70 dtype=int, 

71 default=(0,), 

72 optional=True, 

73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

74 "It will be removed after v20. Use fitBands instead."), 

75 ) 

76 fitBands = pexConfig.ListField( 

77 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

78 "the atmosphere constrained from the 'fitBands' on the same night. " 

79 "Must be a subset of `config.bands`"), 

80 dtype=str, 

81 default=[], 

82 ) 

83 requiredFlag = pexConfig.ListField( 

84 doc=("Flag for which bands are required for a star to be considered a calibration " 

85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must " 

86 "be same length as config.bands, and matched band-by-band."), 

87 dtype=int, 

88 default=(0,), 

89 optional=True, 

90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

91 "It will be removed after v20. Use requiredBands instead."), 

92 ) 

93 requiredBands = pexConfig.ListField( 

94 doc=("Bands that are required for a star to be considered a calibration star. " 

95 "Must be a subset of `config.bands`"), 

96 dtype=str, 

97 default=[], 

98 ) 

99 filterMap = pexConfig.DictField( 

100 doc="Mapping from 'filterName' to band.", 

101 keytype=str, 

102 itemtype=str, 

103 default={}, 

104 ) 

105 doReferenceCalibration = pexConfig.Field( 

106 doc="Use reference catalog as additional constraint on calibration", 

107 dtype=bool, 

108 default=True, 

109 ) 

110 refStarSnMin = pexConfig.Field( 

111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

112 dtype=float, 

113 default=50.0, 

114 ) 

115 refStarOutlierNSig = pexConfig.Field( 

116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

118 dtype=float, 

119 default=4.0, 

120 ) 

121 applyRefStarColorCuts = pexConfig.Field( 

122 doc="Apply color cuts to reference stars?", 

123 dtype=bool, 

124 default=True, 

125 ) 

126 nCore = pexConfig.Field( 

127 doc="Number of cores to use", 

128 dtype=int, 

129 default=4, 

130 ) 

131 nStarPerRun = pexConfig.Field( 

132 doc="Number of stars to run in each chunk", 

133 dtype=int, 

134 default=200000, 

135 ) 

136 nExpPerRun = pexConfig.Field( 

137 doc="Number of exposures to run in each chunk", 

138 dtype=int, 

139 default=1000, 

140 ) 

141 reserveFraction = pexConfig.Field( 

142 doc="Fraction of stars to reserve for testing", 

143 dtype=float, 

144 default=0.1, 

145 ) 

146 freezeStdAtmosphere = pexConfig.Field( 

147 doc="Freeze atmosphere parameters to standard (for testing)", 

148 dtype=bool, 

149 default=False, 

150 ) 

151 precomputeSuperStarInitialCycle = pexConfig.Field( 

152 doc="Precompute superstar flat for initial cycle", 

153 dtype=bool, 

154 default=False, 

155 ) 

156 superStarSubCcd = pexConfig.Field( 

157 doc="Compute superstar flat on sub-ccd scale", 

158 dtype=bool, 

159 default=True, 

160 optional=True, 

161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

162 "It will be removed after v20. Use superStarSubCcdDict instead."), 

163 ) 

164 superStarSubCcdDict = pexConfig.DictField( 

165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

166 "Must have one entry per band."), 

167 keytype=str, 

168 itemtype=bool, 

169 default={}, 

170 ) 

171 superStarSubCcdChebyshevOrder = pexConfig.Field( 

172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

173 "Global default is first-order polynomials, and should be overridden " 

174 "on a camera-by-camera basis depending on the ISR."), 

175 dtype=int, 

176 default=1, 

177 ) 

178 superStarSubCcdTriangular = pexConfig.Field( 

179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

180 "suppress high-order cross terms?"), 

181 dtype=bool, 

182 default=False, 

183 ) 

184 superStarSigmaClip = pexConfig.Field( 

185 doc="Number of sigma to clip outliers when selecting for superstar flats", 

186 dtype=float, 

187 default=5.0, 

188 ) 

189 ccdGraySubCcd = pexConfig.Field( 

190 doc="Compute CCD gray terms on sub-ccd scale", 

191 dtype=bool, 

192 default=False, 

193 optional=True, 

194 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

195 "It will be removed after v20. Use ccdGraySubCcdDict instead."), 

196 ) 

197 ccdGraySubCcdDict = pexConfig.DictField( 

198 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

199 "('ccd gray') on a sub-ccd scale."), 

200 keytype=str, 

201 itemtype=bool, 

202 default={}, 

203 ) 

204 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

205 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

206 dtype=int, 

207 default=1, 

208 ) 

209 ccdGraySubCcdTriangular = pexConfig.Field( 

210 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

211 "suppress high-order cross terms?"), 

212 dtype=bool, 

213 default=True, 

214 ) 

215 cycleNumber = pexConfig.Field( 

216 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

217 "and stage of outlier rejection. See cookbook for details."), 

218 dtype=int, 

219 default=None, 

220 ) 

221 isFinalCycle = pexConfig.Field( 

222 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

223 "selection of stars and photometric exposures, and will output zeropoints " 

224 "and standard stars for use in fgcmOutputProducts"), 

225 dtype=bool, 

226 default=False, 

227 ) 

228 maxIterBeforeFinalCycle = pexConfig.Field( 

229 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

230 "will always be 0 in the final cycle for cleanup and final selection."), 

231 dtype=int, 

232 default=50, 

233 ) 

234 utBoundary = pexConfig.Field( 

235 doc="Boundary (in UTC) from day-to-day", 

236 dtype=float, 

237 default=None, 

238 ) 

239 washMjds = pexConfig.ListField( 

240 doc="Mirror wash MJDs", 

241 dtype=float, 

242 default=(0.0,), 

243 ) 

244 epochMjds = pexConfig.ListField( 

245 doc="Epoch boundaries in MJD", 

246 dtype=float, 

247 default=(0.0,), 

248 ) 

249 minObsPerBand = pexConfig.Field( 

250 doc="Minimum good observations per band", 

251 dtype=int, 

252 default=2, 

253 ) 

254 # TODO: When DM-16511 is done, it will be possible to get the 

255 # telescope latitude directly from the camera. 

256 latitude = pexConfig.Field( 

257 doc="Observatory latitude", 

258 dtype=float, 

259 default=None, 

260 ) 

261 brightObsGrayMax = pexConfig.Field( 

262 doc="Maximum gray extinction to be considered bright observation", 

263 dtype=float, 

264 default=0.15, 

265 ) 

266 minStarPerCcd = pexConfig.Field( 

267 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

268 "CCDs with fewer stars will have their calibration estimated from other " 

269 "CCDs in the same visit, with zeropoint error increased accordingly."), 

270 dtype=int, 

271 default=5, 

272 ) 

273 minCcdPerExp = pexConfig.Field( 

274 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

275 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

276 dtype=int, 

277 default=5, 

278 ) 

279 maxCcdGrayErr = pexConfig.Field( 

280 doc="Maximum error on CCD gray offset to be considered photometric", 

281 dtype=float, 

282 default=0.05, 

283 ) 

284 minStarPerExp = pexConfig.Field( 

285 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

286 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

287 dtype=int, 

288 default=600, 

289 ) 

290 minExpPerNight = pexConfig.Field( 

291 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

292 dtype=int, 

293 default=10, 

294 ) 

295 expGrayInitialCut = pexConfig.Field( 

296 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

297 "observations."), 

298 dtype=float, 

299 default=-0.25, 

300 ) 

301 expGrayPhotometricCut = pexConfig.ListField( 

302 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. " 

303 "Must be same length as config.bands, and matched band-by-band."), 

304 dtype=float, 

305 default=(0.0,), 

306 optional=True, 

307 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

308 "It will be removed after v20. Use expGrayPhotometricCutDict instead."), 

309 ) 

310 expGrayPhotometricCutDict = pexConfig.DictField( 

311 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

312 "('gray term') for a visit to be considered photometric. Must have one " 

313 "entry per band. Broad-band filters should be -0.05."), 

314 keytype=str, 

315 itemtype=float, 

316 default={}, 

317 ) 

318 expGrayHighCut = pexConfig.ListField( 

319 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. " 

320 "Must be same length as config.bands, and matched band-by-band."), 

321 dtype=float, 

322 default=(0.0,), 

323 optional=True, 

324 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

325 "It will be removed after v20. Use expGrayHighCutDict instead."), 

326 ) 

327 expGrayHighCutDict = pexConfig.DictField( 

328 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

329 "('gray term') for a visit to be considered photometric. Must have one " 

330 "entry per band. Broad-band filters should be 0.2."), 

331 keytype=str, 

332 itemtype=float, 

333 default={}, 

334 ) 

335 expGrayRecoverCut = pexConfig.Field( 

336 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

337 "Visits with more gray extinction will only get CCD zeropoints if there are " 

338 "sufficient star observations (minStarPerCcd) on that CCD."), 

339 dtype=float, 

340 default=-1.0, 

341 ) 

342 expVarGrayPhotometricCut = pexConfig.Field( 

343 doc="Maximum exposure variance to be considered possibly photometric", 

344 dtype=float, 

345 default=0.0005, 

346 optional=True, 

347 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

348 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."), 

349 ) 

350 expVarGrayPhotometricCutDict = pexConfig.DictField( 

351 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

352 "photometric. Must have one entry per band. Broad-band filters should be " 

353 "0.0005."), 

354 keytype=str, 

355 itemtype=float, 

356 default={}, 

357 ) 

358 expGrayErrRecoverCut = pexConfig.Field( 

359 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

360 "Visits with more gray variance will only get CCD zeropoints if there are " 

361 "sufficient star observations (minStarPerCcd) on that CCD."), 

362 dtype=float, 

363 default=0.05, 

364 ) 

365 aperCorrFitNBins = pexConfig.Field( 

366 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

367 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

368 "used if available."), 

369 dtype=int, 

370 default=10, 

371 ) 

372 aperCorrInputSlopes = pexConfig.ListField( 

373 doc=("Aperture correction input slope parameters. These are used on the first " 

374 "fit iteration, and aperture correction parameters will be updated from " 

375 "the data if config.aperCorrFitNBins > 0. It is recommended to set this" 

376 "when there is insufficient data to fit the parameters (e.g. tract mode). " 

377 "If set, must be same length as config.bands, and matched band-by-band."), 

378 dtype=float, 

379 default=[], 

380 optional=True, 

381 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

382 "It will be removed after v20. Use aperCorrInputSlopeDict instead."), 

383 ) 

384 aperCorrInputSlopeDict = pexConfig.DictField( 

385 doc=("Per-band specification of aperture correction input slope parameters. These " 

386 "are used on the first fit iteration, and aperture correction parameters will " 

387 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

388 "to set this when there is insufficient data to fit the parameters (e.g. " 

389 "tract mode)."), 

390 keytype=str, 

391 itemtype=float, 

392 default={}, 

393 ) 

394 sedFudgeFactors = pexConfig.ListField( 

395 doc=("Fudge factors for computing linear SED from colors. Must be same length as " 

396 "config.bands, and matched band-by-band."), 

397 dtype=float, 

398 default=(0,), 

399 optional=True, 

400 deprecated=("This field has been deprecated and will be removed after v20. " 

401 "Please use sedSlopeTermMap and sedSlopeMap."), 

402 ) 

403 sedboundaryterms = pexConfig.ConfigField( 

404 doc="Mapping from bands to SED boundary term names used is sedterms.", 

405 dtype=SedboundarytermDict, 

406 ) 

407 sedterms = pexConfig.ConfigField( 

408 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

409 dtype=SedtermDict, 

410 ) 

411 sigFgcmMaxErr = pexConfig.Field( 

412 doc="Maximum mag error for fitting sigma_FGCM", 

413 dtype=float, 

414 default=0.01, 

415 ) 

416 sigFgcmMaxEGray = pexConfig.ListField( 

417 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. " 

418 "May be 1 element (same for all bands) or the same length as config.bands."), 

419 dtype=float, 

420 default=(0.05,), 

421 optional=True, 

422 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

423 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."), 

424 ) 

425 sigFgcmMaxEGrayDict = pexConfig.DictField( 

426 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

427 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

428 "should be 0.05."), 

429 keytype=str, 

430 itemtype=float, 

431 default={}, 

432 ) 

433 ccdGrayMaxStarErr = pexConfig.Field( 

434 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

435 "computation"), 

436 dtype=float, 

437 default=0.10, 

438 ) 

439 approxThroughput = pexConfig.ListField( 

440 doc=("Approximate overall throughput at start of calibration observations. " 

441 "May be 1 element (same for all bands) or the same length as config.bands."), 

442 dtype=float, 

443 default=(1.0, ), 

444 optional=True, 

445 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

446 "It will be removed after v20. Use approxThroughputDict instead."), 

447 ) 

448 approxThroughputDict = pexConfig.DictField( 

449 doc=("Per-band specification of the approximate overall throughput at the start of " 

450 "calibration observations. Must have one entry per band. Typically should " 

451 "be 1.0."), 

452 keytype=str, 

453 itemtype=float, 

454 default={}, 

455 ) 

456 sigmaCalRange = pexConfig.ListField( 

457 doc="Allowed range for systematic error floor estimation", 

458 dtype=float, 

459 default=(0.001, 0.003), 

460 ) 

461 sigmaCalFitPercentile = pexConfig.ListField( 

462 doc="Magnitude percentile range to fit systematic error floor", 

463 dtype=float, 

464 default=(0.05, 0.15), 

465 ) 

466 sigmaCalPlotPercentile = pexConfig.ListField( 

467 doc="Magnitude percentile range to plot systematic error floor", 

468 dtype=float, 

469 default=(0.05, 0.95), 

470 ) 

471 sigma0Phot = pexConfig.Field( 

472 doc="Systematic error floor for all zeropoints", 

473 dtype=float, 

474 default=0.003, 

475 ) 

476 mapLongitudeRef = pexConfig.Field( 

477 doc="Reference longitude for plotting maps", 

478 dtype=float, 

479 default=0.0, 

480 ) 

481 mapNSide = pexConfig.Field( 

482 doc="Healpix nside for plotting maps", 

483 dtype=int, 

484 default=256, 

485 ) 

486 outfileBase = pexConfig.Field( 

487 doc="Filename start for plot output files", 

488 dtype=str, 

489 default=None, 

490 ) 

491 starColorCuts = pexConfig.ListField( 

492 doc="Encoded star-color cuts (to be cleaned up)", 

493 dtype=str, 

494 default=("NO_DATA",), 

495 ) 

496 colorSplitIndices = pexConfig.ListField( 

497 doc="Band indices to use to split stars by color", 

498 dtype=int, 

499 default=None, 

500 optional=True, 

501 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

502 "It will be removed after v20. Use colorSplitBands instead."), 

503 ) 

504 colorSplitBands = pexConfig.ListField( 

505 doc="Band names to use to split stars by color. Must have 2 entries.", 

506 dtype=str, 

507 length=2, 

508 default=('g', 'i'), 

509 ) 

510 modelMagErrors = pexConfig.Field( 

511 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

512 dtype=bool, 

513 default=True, 

514 ) 

515 useQuadraticPwv = pexConfig.Field( 

516 doc="Model PWV with a quadratic term for variation through the night?", 

517 dtype=bool, 

518 default=False, 

519 ) 

520 instrumentParsPerBand = pexConfig.Field( 

521 doc=("Model instrumental parameters per band? " 

522 "Otherwise, instrumental parameters (QE changes with time) are " 

523 "shared among all bands."), 

524 dtype=bool, 

525 default=False, 

526 ) 

527 instrumentSlopeMinDeltaT = pexConfig.Field( 

528 doc=("Minimum time change (in days) between observations to use in constraining " 

529 "instrument slope."), 

530 dtype=float, 

531 default=20.0, 

532 ) 

533 fitMirrorChromaticity = pexConfig.Field( 

534 doc="Fit (intraband) mirror chromatic term?", 

535 dtype=bool, 

536 default=False, 

537 ) 

538 coatingMjds = pexConfig.ListField( 

539 doc="Mirror coating dates in MJD", 

540 dtype=float, 

541 default=(0.0,), 

542 ) 

543 outputStandardsBeforeFinalCycle = pexConfig.Field( 

544 doc="Output standard stars prior to final cycle? Used in debugging.", 

545 dtype=bool, 

546 default=False, 

547 ) 

548 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

549 doc="Output standard stars prior to final cycle? Used in debugging.", 

550 dtype=bool, 

551 default=False, 

552 ) 

553 useRepeatabilityForExpGrayCuts = pexConfig.ListField( 

554 doc=("Use star repeatability (instead of exposures) for computing photometric " 

555 "cuts? Recommended for tract mode or bands with few exposures. " 

556 "May be 1 element (same for all bands) or the same length as config.bands."), 

557 dtype=bool, 

558 default=(False,), 

559 optional=True, 

560 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

561 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."), 

562 ) 

563 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

564 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

565 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

566 keytype=str, 

567 itemtype=bool, 

568 default={}, 

569 ) 

570 autoPhotometricCutNSig = pexConfig.Field( 

571 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

572 "Cut is based on exposure gray width (per band), unless " 

573 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

574 "repeatability is used (also per band)."), 

575 dtype=float, 

576 default=3.0, 

577 ) 

578 autoHighCutNSig = pexConfig.Field( 

579 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

580 "Cut is based on exposure gray width (per band), unless " 

581 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

582 "repeatability is used (also per band)."), 

583 dtype=float, 

584 default=4.0, 

585 ) 

586 quietMode = pexConfig.Field( 

587 doc="Be less verbose with logging.", 

588 dtype=bool, 

589 default=False, 

590 ) 

591 

592 def setDefaults(self): 

593 pass 

594 

595 def validate(self): 

596 super().validate() 

597 

598 for band in self.fitBands: 

599 if band not in self.bands: 

600 msg = 'fitBand %s not in bands' % (band) 

601 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

602 for band in self.requiredBands: 

603 if band not in self.bands: 

604 msg = 'requiredBand %s not in bands' % (band) 

605 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

606 for band in self.colorSplitBands: 

607 if band not in self.bands: 

608 msg = 'colorSplitBand %s not in bands' % (band) 

609 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

610 for band in self.bands: 

611 if band not in self.superStarSubCcdDict: 

612 msg = 'band %s not in superStarSubCcdDict' % (band) 

613 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

614 self, msg) 

615 if band not in self.ccdGraySubCcdDict: 

616 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

617 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

618 self, msg) 

619 if band not in self.expGrayPhotometricCutDict: 

620 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

621 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

622 self, msg) 

623 if band not in self.expGrayHighCutDict: 

624 msg = 'band %s not in expGrayHighCutDict' % (band) 

625 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

626 self, msg) 

627 if band not in self.expVarGrayPhotometricCutDict: 

628 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

629 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

630 self, msg) 

631 if band not in self.sigFgcmMaxEGrayDict: 

632 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

633 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

634 self, msg) 

635 if band not in self.approxThroughputDict: 

636 msg = 'band %s not in approxThroughputDict' % (band) 

637 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

638 self, msg) 

639 if band not in self.useRepeatabilityForExpGrayCutsDict: 

640 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

641 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

642 self, msg) 

643 

644 

645class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner): 

646 """Subclass of TaskRunner for fgcmFitCycleTask 

647 

648 fgcmFitCycleTask.run() takes one argument, the butler, and uses 

649 stars and visits previously extracted from dataRefs by 

650 fgcmBuildStars. 

651 This Runner does not perform any dataRef parallelization, but the FGCM 

652 code called by the Task uses python multiprocessing (see the "ncores" 

653 config option). 

654 """ 

655 

656 @staticmethod 

657 def getTargetList(parsedCmd): 

658 """ 

659 Return a list with one element, the butler. 

660 """ 

661 return [parsedCmd.butler] 

662 

663 def __call__(self, butler): 

664 """ 

665 Parameters 

666 ---------- 

667 butler: `lsst.daf.persistence.Butler` 

668 

669 Returns 

670 ------- 

671 exitStatus: `list` with `pipeBase.Struct` 

672 exitStatus (0: success; 1: failure) 

673 """ 

674 

675 task = self.TaskClass(config=self.config, log=self.log) 

676 

677 exitStatus = 0 

678 if self.doRaise: 

679 task.runDataRef(butler) 

680 else: 

681 try: 

682 task.runDataRef(butler) 

683 except Exception as e: 

684 exitStatus = 1 

685 task.log.fatal("Failed: %s" % e) 

686 if not isinstance(e, pipeBase.TaskError): 

687 traceback.print_exc(file=sys.stderr) 

688 

689 task.writeMetadata(butler) 

690 

691 # The task does not return any results: 

692 return [pipeBase.Struct(exitStatus=exitStatus)] 

693 

694 def run(self, parsedCmd): 

695 """ 

696 Run the task, with no multiprocessing 

697 

698 Parameters 

699 ---------- 

700 parsedCmd: ArgumentParser parsed command line 

701 """ 

702 

703 resultList = [] 

704 

705 if self.precall(parsedCmd): 

706 targetList = self.getTargetList(parsedCmd) 

707 # make sure that we only get 1 

708 resultList = self(targetList[0]) 

709 

710 return resultList 

711 

712 

713class FgcmFitCycleTask(pipeBase.CmdLineTask): 

714 """ 

715 Run Single fit cycle for FGCM global calibration 

716 """ 

717 

718 ConfigClass = FgcmFitCycleConfig 

719 RunnerClass = FgcmFitCycleRunner 

720 _DefaultName = "fgcmFitCycle" 

721 

722 def __init__(self, butler=None, **kwargs): 

723 """ 

724 Instantiate an fgcmFitCycle. 

725 

726 Parameters 

727 ---------- 

728 butler : `lsst.daf.persistence.Butler` 

729 """ 

730 

731 pipeBase.CmdLineTask.__init__(self, **kwargs) 

732 

733 # no saving of metadata for now 

734 def _getMetadataName(self): 

735 return None 

736 

737 @pipeBase.timeMethod 

738 def runDataRef(self, butler): 

739 """ 

740 Run a single fit cycle for FGCM 

741 

742 Parameters 

743 ---------- 

744 butler: `lsst.daf.persistence.Butler` 

745 """ 

746 

747 self._fgcmFitCycle(butler) 

748 

749 def writeConfig(self, butler, clobber=False, doBackup=True): 

750 """Write the configuration used for processing the data, or check that an existing 

751 one is equal to the new one if present. This is an override of the regular 

752 version from pipe_base that knows about fgcmcycle. 

753 

754 Parameters 

755 ---------- 

756 butler : `lsst.daf.persistence.Butler` 

757 Data butler used to write the config. The config is written to dataset type 

758 `CmdLineTask._getConfigName`. 

759 clobber : `bool`, optional 

760 A boolean flag that controls what happens if a config already has been saved: 

761 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 

762 - `False`: raise `TaskError` if this config does not match the existing config. 

763 doBackup : `bool`, optional 

764 Set to `True` to backup the config files if clobbering. 

765 """ 

766 configName = self._getConfigName() 

767 if configName is None: 

768 return 

769 if clobber: 

770 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber) 

771 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber): 

772 # this may be subject to a race condition; see #2789 

773 try: 

774 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber) 

775 except Exception as exc: 

776 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" % 

777 (configName, exc)) 

778 

779 def logConfigMismatch(msg): 

780 self.log.fatal("Comparing configuration: %s", msg) 

781 

782 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

783 raise pipeBase.TaskError( 

784 ("Config does not match existing task config %r on disk; tasks configurations " + 

785 "must be consistent within the same output repo (override with --clobber-config)") % 

786 (configName,)) 

787 else: 

788 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber) 

789 

790 def _fgcmFitCycle(self, butler): 

791 """ 

792 Run the fit cycle 

793 

794 Parameters 

795 ---------- 

796 butler: `lsst.daf.persistence.Butler` 

797 """ 

798 

799 self._checkDatasetsExist(butler) 

800 

801 # Set defaults on whether to output standards and zeropoints 

802 self.maxIter = self.config.maxIterBeforeFinalCycle 

803 self.outputStandards = self.config.outputStandardsBeforeFinalCycle 

804 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle 

805 self.resetFitParameters = True 

806 

807 if self.config.isFinalCycle: 

808 # This is the final fit cycle, so we do not want to reset fit 

809 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

810 # and we always want to output standards and zeropoints 

811 self.maxIter = 0 

812 self.outputStandards = True 

813 self.outputZeropoints = True 

814 self.resetFitParameters = False 

815 

816 camera = butler.get('camera') 

817 configDict = makeConfigDict(self.config, self.log, camera, 

818 self.maxIter, self.resetFitParameters, 

819 self.outputZeropoints) 

820 

821 lutCat = butler.get('fgcmLookUpTable') 

822 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap)) 

823 del lutCat 

824 

825 # next we need the exposure/visit information 

826 

827 # fgcmExpInfo = self._loadVisitCatalog(butler) 

828 visitCat = butler.get('fgcmVisitCatalog') 

829 fgcmExpInfo = translateVisitCatalog(visitCat) 

830 del visitCat 

831 

832 # Use the first orientation. 

833 # TODO: DM-21215 will generalize to arbitrary camera orientations 

834 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

835 

836 noFitsDict = {'lutIndex': lutIndexVals, 

837 'lutStd': lutStd, 

838 'expInfo': fgcmExpInfo, 

839 'ccdOffsets': ccdOffsets} 

840 

841 # set up the fitter object 

842 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

843 noFitsDict=noFitsDict, noOutput=True) 

844 

845 # create the parameter object 

846 if (fgcmFitCycle.initialCycle): 

847 # cycle = 0, initial cycle 

848 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

849 fgcmLut, 

850 fgcmExpInfo) 

851 else: 

852 inParInfo, inParams, inSuperStar = self._loadParameters(butler) 

853 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

854 fgcmExpInfo, 

855 inParInfo, 

856 inParams, 

857 inSuperStar) 

858 

859 lastCycle = configDict['cycleNumber'] - 1 

860 

861 # set up the stars... 

862 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

863 

864 starObs = butler.get('fgcmStarObservations') 

865 starIds = butler.get('fgcmStarIds') 

866 starIndices = butler.get('fgcmStarIndices') 

867 

868 # grab the flagged stars if available 

869 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle): 

870 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle) 

871 flagId = flaggedStars['objId'][:] 

872 flagFlag = flaggedStars['objFlag'][:] 

873 else: 

874 flagId = None 

875 flagFlag = None 

876 

877 if self.config.doReferenceCalibration: 

878 refStars = butler.get('fgcmReferenceStars') 

879 

880 refMag, refMagErr = extractReferenceMags(refStars, 

881 self.config.bands, 

882 self.config.filterMap) 

883 refId = refStars['fgcm_id'][:] 

884 else: 

885 refId = None 

886 refMag = None 

887 refMagErr = None 

888 

889 # match star observations to visits 

890 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

891 # actually be transferred into fgcm using the indexing below. 

892 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

893 

894 # The fgcmStars.loadStars method will copy all the star information into 

895 # special shared memory objects that will not blow up the memory usage when 

896 # used with python multiprocessing. Once all the numbers are copied, 

897 # it is necessary to release all references to the objects that previously 

898 # stored the data to ensure that the garbage collector can clear the memory, 

899 # and ensure that this memory is not copied when multiprocessing kicks in. 

900 

901 # We determine the conversion from the native units (typically radians) to 

902 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

903 # numpy arrays rather than Angles, which would we approximately 600x slower. 

904 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

905 

906 fgcmStars.loadStars(fgcmPars, 

907 starObs['visit'][starIndices['obsIndex']], 

908 starObs['ccd'][starIndices['obsIndex']], 

909 starObs['ra'][starIndices['obsIndex']] * conv, 

910 starObs['dec'][starIndices['obsIndex']] * conv, 

911 starObs['instMag'][starIndices['obsIndex']], 

912 starObs['instMagErr'][starIndices['obsIndex']], 

913 fgcmExpInfo['FILTERNAME'][visitIndex], 

914 starIds['fgcm_id'][:], 

915 starIds['ra'][:], 

916 starIds['dec'][:], 

917 starIds['obsArrIndex'][:], 

918 starIds['nObs'][:], 

919 obsX=starObs['x'][starIndices['obsIndex']], 

920 obsY=starObs['y'][starIndices['obsIndex']], 

921 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

922 refID=refId, 

923 refMag=refMag, 

924 refMagErr=refMagErr, 

925 flagID=flagId, 

926 flagFlag=flagFlag, 

927 computeNobs=True) 

928 

929 # Release all references to temporary objects holding star data (see above) 

930 starObs = None 

931 starIds = None 

932 starIndices = None 

933 flagId = None 

934 flagFlag = None 

935 flaggedStars = None 

936 refStars = None 

937 

938 # and set the bits in the cycle object 

939 fgcmFitCycle.setLUT(fgcmLut) 

940 fgcmFitCycle.setStars(fgcmStars) 

941 fgcmFitCycle.setPars(fgcmPars) 

942 

943 # finish the setup 

944 fgcmFitCycle.finishSetup() 

945 

946 # and run 

947 fgcmFitCycle.run() 

948 

949 ################## 

950 # Persistance 

951 ################## 

952 

953 self._persistFgcmDatasets(butler, fgcmFitCycle) 

954 

955 # Output the config for the next cycle 

956 # We need to make a copy since the input one has been frozen 

957 

958 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

959 i, b in enumerate(self.config.bands)} 

960 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

961 i, band in enumerate(self.config.bands)} 

962 

963 outConfig = copy.copy(self.config) 

964 outConfig.update(cycleNumber=(self.config.cycleNumber + 1), 

965 precomputeSuperStarInitialCycle=False, 

966 freezeStdAtmosphere=False, 

967 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

968 expGrayHighCutDict=updatedHighCutDict) 

969 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

970 outConfig.cycleNumber) 

971 outConfig.save(configFileName) 

972 

973 if self.config.isFinalCycle == 1: 

974 # We are done, ready to output products 

975 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

976 else: 

977 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

978 self.log.info("Be sure to look at:") 

979 self.log.info(" config.expGrayPhotometricCut") 

980 self.log.info(" config.expGrayHighCut") 

981 self.log.info("If you are satisfied with the fit, please set:") 

982 self.log.info(" config.isFinalCycle = True") 

983 

984 def _checkDatasetsExist(self, butler): 

985 """ 

986 Check if necessary datasets exist to run fgcmFitCycle 

987 

988 Parameters 

989 ---------- 

990 butler: `lsst.daf.persistence.Butler` 

991 

992 Raises 

993 ------ 

994 RuntimeError 

995 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 

996 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 

997 If cycleNumber > 0, then also checks for fgcmFitParameters, 

998 fgcmFlaggedStars. 

999 """ 

1000 

1001 if not butler.datasetExists('fgcmVisitCatalog'): 

1002 raise RuntimeError("Could not find fgcmVisitCatalog in repo!") 

1003 if not butler.datasetExists('fgcmStarObservations'): 

1004 raise RuntimeError("Could not find fgcmStarObservations in repo!") 

1005 if not butler.datasetExists('fgcmStarIds'): 

1006 raise RuntimeError("Could not find fgcmStarIds in repo!") 

1007 if not butler.datasetExists('fgcmStarIndices'): 

1008 raise RuntimeError("Could not find fgcmStarIndices in repo!") 

1009 if not butler.datasetExists('fgcmLookUpTable'): 

1010 raise RuntimeError("Could not find fgcmLookUpTable in repo!") 

1011 

1012 # Need additional datasets if we are not the initial cycle 

1013 if (self.config.cycleNumber > 0): 

1014 if not butler.datasetExists('fgcmFitParameters', 

1015 fgcmcycle=self.config.cycleNumber-1): 

1016 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" % 

1017 (self.config.cycleNumber-1)) 

1018 if not butler.datasetExists('fgcmFlaggedStars', 

1019 fgcmcycle=self.config.cycleNumber-1): 

1020 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" % 

1021 (self.config.cycleNumber-1)) 

1022 

1023 # And additional dataset if we want reference calibration 

1024 if self.config.doReferenceCalibration: 

1025 if not butler.datasetExists('fgcmReferenceStars'): 

1026 raise RuntimeError("Could not find fgcmReferenceStars in repo, and " 

1027 "doReferenceCalibration is True.") 

1028 

1029 def _loadParameters(self, butler): 

1030 """ 

1031 Load FGCM parameters from a previous fit cycle 

1032 

1033 Parameters 

1034 ---------- 

1035 butler: `lsst.daf.persistence.Butler` 

1036 

1037 Returns 

1038 ------- 

1039 inParInfo: `numpy.ndarray` 

1040 Numpy array parameter information formatted for input to fgcm 

1041 inParameters: `numpy.ndarray` 

1042 Numpy array parameter values formatted for input to fgcm 

1043 inSuperStar: `numpy.array` 

1044 Superstar flat formatted for input to fgcm 

1045 """ 

1046 

1047 # note that we already checked that this is available 

1048 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1) 

1049 

1050 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1051 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1052 parNotFitBands = np.array(parCat[0]['notFitBands'].split(',')) 

1053 

1054 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1055 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1056 (parLutFilterNames.size, )), 

1057 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1058 ('NOTFITBANDS', parNotFitBands.dtype.str, (parNotFitBands.size, )), 

1059 ('LNTAUUNIT', 'f8'), 

1060 ('LNTAUSLOPEUNIT', 'f8'), 

1061 ('ALPHAUNIT', 'f8'), 

1062 ('LNPWVUNIT', 'f8'), 

1063 ('LNPWVSLOPEUNIT', 'f8'), 

1064 ('LNPWVQUADRATICUNIT', 'f8'), 

1065 ('LNPWVGLOBALUNIT', 'f8'), 

1066 ('O3UNIT', 'f8'), 

1067 ('QESYSUNIT', 'f8'), 

1068 ('FILTEROFFSETUNIT', 'f8'), 

1069 ('HASEXTERNALPWV', 'i2'), 

1070 ('HASEXTERNALTAU', 'i2')]) 

1071 inParInfo['NCCD'] = parCat['nCcd'] 

1072 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1073 inParInfo['FITBANDS'][:] = parFitBands 

1074 inParInfo['NOTFITBANDS'][:] = parNotFitBands 

1075 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1076 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1077 

1078 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1079 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1080 ('PARLNTAUINTERCEPT', 'f8', 

1081 (parCat['parLnTauIntercept'].size, )), 

1082 ('PARLNTAUSLOPE', 'f8', 

1083 (parCat['parLnTauSlope'].size, )), 

1084 ('PARLNPWVINTERCEPT', 'f8', 

1085 (parCat['parLnPwvIntercept'].size, )), 

1086 ('PARLNPWVSLOPE', 'f8', 

1087 (parCat['parLnPwvSlope'].size, )), 

1088 ('PARLNPWVQUADRATIC', 'f8', 

1089 (parCat['parLnPwvQuadratic'].size, )), 

1090 ('PARQESYSINTERCEPT', 'f8', 

1091 (parCat['parQeSysIntercept'].size, )), 

1092 ('COMPQESYSSLOPE', 'f8', 

1093 (parCat['compQeSysSlope'].size, )), 

1094 ('PARFILTEROFFSET', 'f8', 

1095 (parCat['parFilterOffset'].size, )), 

1096 ('PARFILTEROFFSETFITFLAG', 'i2', 

1097 (parCat['parFilterOffsetFitFlag'].size, )), 

1098 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1099 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1100 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1101 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1102 ('COMPABSTHROUGHPUT', 'f8', 

1103 (parCat['compAbsThroughput'].size, )), 

1104 ('COMPREFOFFSET', 'f8', 

1105 (parCat['compRefOffset'].size, )), 

1106 ('COMPREFSIGMA', 'f8', 

1107 (parCat['compRefSigma'].size, )), 

1108 ('COMPMIRRORCHROMATICITY', 'f8', 

1109 (parCat['compMirrorChromaticity'].size, )), 

1110 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1111 (parCat['mirrorChromaticityPivot'].size, )), 

1112 ('COMPAPERCORRPIVOT', 'f8', 

1113 (parCat['compAperCorrPivot'].size, )), 

1114 ('COMPAPERCORRSLOPE', 'f8', 

1115 (parCat['compAperCorrSlope'].size, )), 

1116 ('COMPAPERCORRSLOPEERR', 'f8', 

1117 (parCat['compAperCorrSlopeErr'].size, )), 

1118 ('COMPAPERCORRRANGE', 'f8', 

1119 (parCat['compAperCorrRange'].size, )), 

1120 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1121 (parCat['compModelErrExptimePivot'].size, )), 

1122 ('COMPMODELERRFWHMPIVOT', 'f8', 

1123 (parCat['compModelErrFwhmPivot'].size, )), 

1124 ('COMPMODELERRSKYPIVOT', 'f8', 

1125 (parCat['compModelErrSkyPivot'].size, )), 

1126 ('COMPMODELERRPARS', 'f8', 

1127 (parCat['compModelErrPars'].size, )), 

1128 ('COMPEXPGRAY', 'f8', 

1129 (parCat['compExpGray'].size, )), 

1130 ('COMPVARGRAY', 'f8', 

1131 (parCat['compVarGray'].size, )), 

1132 ('COMPNGOODSTARPEREXP', 'i4', 

1133 (parCat['compNGoodStarPerExp'].size, )), 

1134 ('COMPSIGFGCM', 'f8', 

1135 (parCat['compSigFgcm'].size, )), 

1136 ('COMPSIGMACAL', 'f8', 

1137 (parCat['compSigmaCal'].size, )), 

1138 ('COMPRETRIEVEDLNPWV', 'f8', 

1139 (parCat['compRetrievedLnPwv'].size, )), 

1140 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1141 (parCat['compRetrievedLnPwvRaw'].size, )), 

1142 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1143 (parCat['compRetrievedLnPwvFlag'].size, )), 

1144 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1145 (parCat['compRetrievedTauNight'].size, ))]) 

1146 

1147 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1148 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1149 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1150 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1151 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1152 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1153 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1154 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1155 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1156 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1157 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1158 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1159 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1160 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1161 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1162 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1163 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1164 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1165 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1166 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1167 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1168 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1169 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1170 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1171 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1172 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1173 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1174 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1175 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1176 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1177 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1178 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1179 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1180 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1181 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1182 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1183 

1184 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1185 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1186 

1187 return (inParInfo, inParams, inSuperStar) 

1188 

1189 def _persistFgcmDatasets(self, butler, fgcmFitCycle): 

1190 """ 

1191 Persist FGCM datasets through the butler. 

1192 

1193 Parameters 

1194 ---------- 

1195 butler: `lsst.daf.persistence.Butler` 

1196 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1197 Fgcm Fit cycle object 

1198 """ 

1199 

1200 # Save the parameters 

1201 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1202 

1203 parSchema = afwTable.Schema() 

1204 

1205 comma = ',' 

1206 lutFilterNameString = comma.join([n.decode('utf-8') 

1207 for n in parInfo['LUTFILTERNAMES'][0]]) 

1208 fitBandString = comma.join([n.decode('utf-8') 

1209 for n in parInfo['FITBANDS'][0]]) 

1210 notFitBandString = comma.join([n.decode('utf-8') 

1211 for n in parInfo['NOTFITBANDS'][0]]) 

1212 

1213 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1214 lutFilterNameString, fitBandString, notFitBandString) 

1215 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1216 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1217 lutFilterNameString, fitBandString, notFitBandString) 

1218 

1219 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber) 

1220 

1221 # Save the indices of the flagged stars 

1222 # (stars that have been (a) reserved from the fit for testing and 

1223 # (b) bad stars that have failed quality checks.) 

1224 flagStarSchema = self._makeFlagStarSchema() 

1225 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1226 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1227 

1228 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber) 

1229 

1230 # Save the zeropoint information and atmospheres only if desired 

1231 if self.outputZeropoints: 

1232 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1233 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1234 

1235 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1236 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1237 

1238 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber) 

1239 

1240 # Save atmosphere values 

1241 # These are generated by the same code that generates zeropoints 

1242 atmSchema = makeAtmSchema() 

1243 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1244 

1245 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber) 

1246 

1247 # Save the standard stars (if configured) 

1248 if self.outputStandards: 

1249 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1250 stdSchema = makeStdSchema(len(goodBands)) 

1251 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1252 

1253 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber) 

1254 

1255 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1256 lutFilterNameString, fitBandString, notFitBandString): 

1257 """ 

1258 Make the parameter persistence schema 

1259 

1260 Parameters 

1261 ---------- 

1262 parInfo: `numpy.ndarray` 

1263 Parameter information returned by fgcm 

1264 pars: `numpy.ndarray` 

1265 Parameter values returned by fgcm 

1266 parSuperStarFlat: `numpy.array` 

1267 Superstar flat values returned by fgcm 

1268 lutFilterNameString: `str` 

1269 Combined string of all the lutFilterNames 

1270 fitBandString: `str` 

1271 Combined string of all the fitBands 

1272 notFitBandString: `str` 

1273 Combined string of all the bands not used in the fit 

1274 

1275 Returns 

1276 ------- 

1277 parSchema: `afwTable.schema` 

1278 """ 

1279 

1280 parSchema = afwTable.Schema() 

1281 

1282 # parameter info section 

1283 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1284 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1285 size=len(lutFilterNameString)) 

1286 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1287 size=len(fitBandString)) 

1288 parSchema.addField('notFitBands', type=str, doc='Bands that were not fit', 

1289 size=len(notFitBandString)) 

1290 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1291 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1292 doc='Step units for ln(AOD) slope') 

1293 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1294 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1295 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1296 doc='Step units for ln(pwv) slope') 

1297 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1298 doc='Step units for ln(pwv) quadratic term') 

1299 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1300 doc='Step units for global ln(pwv) parameters') 

1301 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1302 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1303 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1304 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1305 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1306 

1307 # parameter section 

1308 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1309 size=pars['PARALPHA'].size) 

1310 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1311 size=pars['PARO3'].size) 

1312 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1313 doc='ln(Tau) intercept parameter vector', 

1314 size=pars['PARLNTAUINTERCEPT'].size) 

1315 parSchema.addField('parLnTauSlope', type='ArrayD', 

1316 doc='ln(Tau) slope parameter vector', 

1317 size=pars['PARLNTAUSLOPE'].size) 

1318 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1319 size=pars['PARLNPWVINTERCEPT'].size) 

1320 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1321 size=pars['PARLNPWVSLOPE'].size) 

1322 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1323 size=pars['PARLNPWVQUADRATIC'].size) 

1324 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1325 size=pars['PARQESYSINTERCEPT'].size) 

1326 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1327 size=pars[0]['COMPQESYSSLOPE'].size) 

1328 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1329 size=pars['PARFILTEROFFSET'].size) 

1330 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1331 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1332 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1333 doc='Global scale for retrieved ln(pwv)') 

1334 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1335 doc='Global offset for retrieved ln(pwv)') 

1336 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1337 doc='Nightly offset for retrieved ln(pwv)', 

1338 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1339 parSchema.addField('compAbsThroughput', type='ArrayD', 

1340 doc='Absolute throughput (relative to transmission curves)', 

1341 size=pars['COMPABSTHROUGHPUT'].size) 

1342 parSchema.addField('compRefOffset', type='ArrayD', 

1343 doc='Offset between reference stars and calibrated stars', 

1344 size=pars['COMPREFOFFSET'].size) 

1345 parSchema.addField('compRefSigma', type='ArrayD', 

1346 doc='Width of reference star/calibrated star distribution', 

1347 size=pars['COMPREFSIGMA'].size) 

1348 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1349 doc='Computed mirror chromaticity terms', 

1350 size=pars['COMPMIRRORCHROMATICITY'].size) 

1351 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1352 doc='Mirror chromaticity pivot mjd', 

1353 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1354 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1355 size=pars['COMPAPERCORRPIVOT'].size) 

1356 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1357 size=pars['COMPAPERCORRSLOPE'].size) 

1358 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1359 size=pars['COMPAPERCORRSLOPEERR'].size) 

1360 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1361 size=pars['COMPAPERCORRRANGE'].size) 

1362 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1363 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1364 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1365 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1366 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1367 size=pars['COMPMODELERRSKYPIVOT'].size) 

1368 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1369 size=pars['COMPMODELERRPARS'].size) 

1370 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1371 size=pars['COMPEXPGRAY'].size) 

1372 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1373 size=pars['COMPVARGRAY'].size) 

1374 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1375 doc='Computed number of good stars per exposure', 

1376 size=pars['COMPNGOODSTARPEREXP'].size) 

1377 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1378 size=pars['COMPSIGFGCM'].size) 

1379 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1380 size=pars['COMPSIGMACAL'].size) 

1381 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1382 size=pars['COMPRETRIEVEDLNPWV'].size) 

1383 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1384 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1385 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1386 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1387 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1388 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1389 # superstarflat section 

1390 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1391 size=4) 

1392 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1393 size=parSuperStarFlat.size) 

1394 

1395 return parSchema 

1396 

1397 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1398 lutFilterNameString, fitBandString, notFitBandString): 

1399 """ 

1400 Make the FGCM parameter catalog for persistence 

1401 

1402 Parameters 

1403 ---------- 

1404 parSchema: `lsst.afw.table.Schema` 

1405 Parameter catalog schema 

1406 pars: `numpy.ndarray` 

1407 FGCM parameters to put into parCat 

1408 parSuperStarFlat: `numpy.array` 

1409 FGCM superstar flat array to put into parCat 

1410 lutFilterNameString: `str` 

1411 Combined string of all the lutFilterNames 

1412 fitBandString: `str` 

1413 Combined string of all the fitBands 

1414 notFitBandString: `str` 

1415 Combined string of all the bands not used in the fit 

1416 

1417 Returns 

1418 ------- 

1419 parCat: `afwTable.BasicCatalog` 

1420 Atmosphere and instrumental model parameter catalog for persistence 

1421 """ 

1422 

1423 parCat = afwTable.BaseCatalog(parSchema) 

1424 parCat.reserve(1) 

1425 

1426 # The parameter catalog just has one row, with many columns for all the 

1427 # atmosphere and instrument fit parameters 

1428 rec = parCat.addNew() 

1429 

1430 # info section 

1431 rec['nCcd'] = parInfo['NCCD'] 

1432 rec['lutFilterNames'] = lutFilterNameString 

1433 rec['fitBands'] = fitBandString 

1434 rec['notFitBands'] = notFitBandString 

1435 # note these are not currently supported here. 

1436 rec['hasExternalPwv'] = 0 

1437 rec['hasExternalTau'] = 0 

1438 

1439 # parameter section 

1440 

1441 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1442 

1443 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1444 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1445 'parQeSysIntercept', 'compQeSysSlope', 

1446 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1447 'parFilterOffset', 'parFilterOffsetFitFlag', 

1448 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1449 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1450 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1451 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1452 'compModelErrSkyPivot', 'compModelErrPars', 

1453 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1454 'compSigmaCal', 

1455 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1456 'compRetrievedTauNight'] 

1457 

1458 for scalarName in scalarNames: 

1459 rec[scalarName] = pars[scalarName.upper()] 

1460 

1461 for arrName in arrNames: 

1462 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1463 

1464 # superstar section 

1465 rec['superstarSize'][:] = parSuperStarFlat.shape 

1466 rec['superstar'][:] = parSuperStarFlat.flatten() 

1467 

1468 return parCat 

1469 

1470 def _makeFlagStarSchema(self): 

1471 """ 

1472 Make the flagged-stars schema 

1473 

1474 Returns 

1475 ------- 

1476 flagStarSchema: `lsst.afw.table.Schema` 

1477 """ 

1478 

1479 flagStarSchema = afwTable.Schema() 

1480 

1481 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1482 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1483 

1484 return flagStarSchema 

1485 

1486 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1487 """ 

1488 Make the flagged star catalog for persistence 

1489 

1490 Parameters 

1491 ---------- 

1492 flagStarSchema: `lsst.afw.table.Schema` 

1493 Flagged star schema 

1494 flagStarStruct: `numpy.ndarray` 

1495 Flagged star structure from fgcm 

1496 

1497 Returns 

1498 ------- 

1499 flagStarCat: `lsst.afw.table.BaseCatalog` 

1500 Flagged star catalog for persistence 

1501 """ 

1502 

1503 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1504 flagStarCat.reserve(flagStarStruct.size) 

1505 for i in range(flagStarStruct.size): 

1506 flagStarCat.addNew() 

1507 

1508 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1509 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1510 

1511 return flagStarCat