Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import sys 

37import traceback 

38import copy 

39 

40import numpy as np 

41 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45import lsst.afw.table as afwTable 

46 

47from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

48from .utilities import extractReferenceMags 

49from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat 

50from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

51from .sedterms import SedboundarytermDict, SedtermDict 

52from .utilities import lookupStaticCalibrations 

53 

54import fgcm 

55 

56__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner'] 

57 

58 

59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("instrument",), 

61 defaultTemplates={"previousCycleNumber": "-1", 

62 "cycleNumber": "0"}): 

63 camera = connectionTypes.PrerequisiteInput( 

64 doc="Camera instrument", 

65 name="camera", 

66 storageClass="Camera", 

67 dimensions=("instrument",), 

68 lookupFunction=lookupStaticCalibrations, 

69 isCalibration=True, 

70 ) 

71 

72 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

73 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

74 "chromatic corrections."), 

75 name="fgcmLookUpTable", 

76 storageClass="Catalog", 

77 dimensions=("instrument",), 

78 deferLoad=True, 

79 ) 

80 

81 fgcmVisitCatalog = connectionTypes.PrerequisiteInput( 

82 doc="Catalog of visit information for fgcm", 

83 name="fgcmVisitCatalog", 

84 storageClass="Catalog", 

85 dimensions=("instrument",), 

86 deferLoad=True, 

87 ) 

88 

89 fgcmStarObservations = connectionTypes.PrerequisiteInput( 

90 doc="Catalog of star observations for fgcm", 

91 name="fgcmStarObservations", 

92 storageClass="Catalog", 

93 dimensions=("instrument",), 

94 deferLoad=True, 

95 ) 

96 

97 fgcmStarIds = connectionTypes.PrerequisiteInput( 

98 doc="Catalog of fgcm calibration star IDs", 

99 name="fgcmStarIds", 

100 storageClass="Catalog", 

101 dimensions=("instrument",), 

102 deferLoad=True, 

103 ) 

104 

105 fgcmStarIndices = connectionTypes.PrerequisiteInput( 

106 doc="Catalog of fgcm calibration star indices", 

107 name="fgcmStarIndices", 

108 storageClass="Catalog", 

109 dimensions=("instrument",), 

110 deferLoad=True, 

111 ) 

112 

113 fgcmReferenceStars = connectionTypes.PrerequisiteInput( 

114 doc="Catalog of fgcm-matched reference stars", 

115 name="fgcmReferenceStars", 

116 storageClass="Catalog", 

117 dimensions=("instrument",), 

118 deferLoad=True, 

119 ) 

120 

121 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput( 

122 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle", 

123 name="fgcmFlaggedStars{previousCycleNumber}", 

124 storageClass="Catalog", 

125 dimensions=("instrument",), 

126 deferLoad=True, 

127 ) 

128 

129 fgcmFitParametersInput = connectionTypes.PrerequisiteInput( 

130 doc="Catalog of fgcm fit parameters from previous fit cycle", 

131 name="fgcmFitParameters{previousCycleNumber}", 

132 storageClass="Catalog", 

133 dimensions=("instrument",), 

134 deferLoad=True, 

135 ) 

136 

137 fgcmFitParameters = connectionTypes.Output( 

138 doc="Catalog of fgcm fit parameters from current fit cycle", 

139 name="fgcmFitParameters{cycleNumber}", 

140 storageClass="Catalog", 

141 dimensions=("instrument",), 

142 ) 

143 

144 fgcmFlaggedStars = connectionTypes.Output( 

145 doc="Catalog of flagged stars for fgcm calibration from current fit cycle", 

146 name="fgcmFlaggedStars{cycleNumber}", 

147 storageClass="Catalog", 

148 dimensions=("instrument",), 

149 ) 

150 

151 fgcmZeropoints = connectionTypes.Output( 

152 doc="Catalog of fgcm zeropoint data from current fit cycle", 

153 name="fgcmZeropoints{cycleNumber}", 

154 storageClass="Catalog", 

155 dimensions=("instrument",), 

156 ) 

157 

158 fgcmAtmosphereParameters = connectionTypes.Output( 

159 doc="Catalog of atmospheric fit parameters from current fit cycle", 

160 name="fgcmAtmosphereParameters{cycleNumber}", 

161 storageClass="Catalog", 

162 dimensions=("instrument",), 

163 ) 

164 

165 fgcmStandardStars = connectionTypes.Output( 

166 doc="Catalog of standard star magnitudes from current fit cycle", 

167 name="fgcmStandardStars{cycleNumber}", 

168 storageClass="SimpleCatalog", 

169 dimensions=("instrument",), 

170 ) 

171 

172 def __init__(self, *, config=None): 

173 super().__init__(config=config) 

174 

175 if not config.doReferenceCalibration: 

176 self.prerequisiteInputs.remove("fgcmReferenceStars") 

177 

178 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

179 raise ValueError("cycleNumber must be of integer format") 

180 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

181 raise ValueError("previousCycleNumber must be of integer format") 

182 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

183 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

184 

185 if int(config.connections.cycleNumber) == 0: 

186 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput") 

187 self.prerequisiteInputs.remove("fgcmFitParametersInput") 

188 

189 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle: 

190 self.outputs.remove("fgcmStandardStars") 

191 

192 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle: 

193 self.outputs.remove("fgcmZeropoints") 

194 self.outputs.remove("fgcmAtmosphereParameters") 

195 

196 

197class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

198 pipelineConnections=FgcmFitCycleConnections): 

199 """Config for FgcmFitCycle""" 

200 

201 bands = pexConfig.ListField( 

202 doc="Bands to run calibration", 

203 dtype=str, 

204 default=[], 

205 ) 

206 fitFlag = pexConfig.ListField( 

207 doc=("Flag for which bands are directly constrained in the FGCM fit. " 

208 "Bands set to 0 will have the atmosphere constrained from observations " 

209 "in other bands on the same night. Must be same length as config.bands, " 

210 "and matched band-by-band."), 

211 dtype=int, 

212 default=(0,), 

213 optional=True, 

214 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

215 "It will be removed after v20. Use fitBands instead."), 

216 ) 

217 fitBands = pexConfig.ListField( 

218 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

219 "the atmosphere constrained from the 'fitBands' on the same night. " 

220 "Must be a subset of `config.bands`"), 

221 dtype=str, 

222 default=[], 

223 ) 

224 requiredFlag = pexConfig.ListField( 

225 doc=("Flag for which bands are required for a star to be considered a calibration " 

226 "star in the FGCM fit. Typically this should be the same as fitFlag. Must " 

227 "be same length as config.bands, and matched band-by-band."), 

228 dtype=int, 

229 default=(0,), 

230 optional=True, 

231 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

232 "It will be removed after v20. Use requiredBands instead."), 

233 ) 

234 requiredBands = pexConfig.ListField( 

235 doc=("Bands that are required for a star to be considered a calibration star. " 

236 "Must be a subset of `config.bands`"), 

237 dtype=str, 

238 default=[], 

239 ) 

240 filterMap = pexConfig.DictField( 

241 doc="Mapping from 'filterName' to band.", 

242 keytype=str, 

243 itemtype=str, 

244 default={}, 

245 ) 

246 doReferenceCalibration = pexConfig.Field( 

247 doc="Use reference catalog as additional constraint on calibration", 

248 dtype=bool, 

249 default=True, 

250 ) 

251 refStarSnMin = pexConfig.Field( 

252 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

253 dtype=float, 

254 default=50.0, 

255 ) 

256 refStarOutlierNSig = pexConfig.Field( 

257 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

258 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

259 dtype=float, 

260 default=4.0, 

261 ) 

262 applyRefStarColorCuts = pexConfig.Field( 

263 doc="Apply color cuts to reference stars?", 

264 dtype=bool, 

265 default=True, 

266 ) 

267 nCore = pexConfig.Field( 

268 doc="Number of cores to use", 

269 dtype=int, 

270 default=4, 

271 ) 

272 nStarPerRun = pexConfig.Field( 

273 doc="Number of stars to run in each chunk", 

274 dtype=int, 

275 default=200000, 

276 ) 

277 nExpPerRun = pexConfig.Field( 

278 doc="Number of exposures to run in each chunk", 

279 dtype=int, 

280 default=1000, 

281 ) 

282 reserveFraction = pexConfig.Field( 

283 doc="Fraction of stars to reserve for testing", 

284 dtype=float, 

285 default=0.1, 

286 ) 

287 freezeStdAtmosphere = pexConfig.Field( 

288 doc="Freeze atmosphere parameters to standard (for testing)", 

289 dtype=bool, 

290 default=False, 

291 ) 

292 precomputeSuperStarInitialCycle = pexConfig.Field( 

293 doc="Precompute superstar flat for initial cycle", 

294 dtype=bool, 

295 default=False, 

296 ) 

297 superStarSubCcd = pexConfig.Field( 

298 doc="Compute superstar flat on sub-ccd scale", 

299 dtype=bool, 

300 default=True, 

301 optional=True, 

302 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

303 "It will be removed after v20. Use superStarSubCcdDict instead."), 

304 ) 

305 superStarSubCcdDict = pexConfig.DictField( 

306 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

307 "Must have one entry per band."), 

308 keytype=str, 

309 itemtype=bool, 

310 default={}, 

311 ) 

312 superStarSubCcdChebyshevOrder = pexConfig.Field( 

313 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

314 "Global default is first-order polynomials, and should be overridden " 

315 "on a camera-by-camera basis depending on the ISR."), 

316 dtype=int, 

317 default=1, 

318 ) 

319 superStarSubCcdTriangular = pexConfig.Field( 

320 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

321 "suppress high-order cross terms?"), 

322 dtype=bool, 

323 default=False, 

324 ) 

325 superStarSigmaClip = pexConfig.Field( 

326 doc="Number of sigma to clip outliers when selecting for superstar flats", 

327 dtype=float, 

328 default=5.0, 

329 ) 

330 focalPlaneSigmaClip = pexConfig.Field( 

331 doc="Number of sigma to clip outliers per focal-plane.", 

332 dtype=float, 

333 default=4.0, 

334 ) 

335 ccdGraySubCcd = pexConfig.Field( 

336 doc="Compute CCD gray terms on sub-ccd scale", 

337 dtype=bool, 

338 default=False, 

339 optional=True, 

340 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

341 "It will be removed after v20. Use ccdGraySubCcdDict instead."), 

342 ) 

343 ccdGraySubCcdDict = pexConfig.DictField( 

344 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

345 "('ccd gray') on a sub-ccd scale."), 

346 keytype=str, 

347 itemtype=bool, 

348 default={}, 

349 ) 

350 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

351 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

352 dtype=int, 

353 default=1, 

354 ) 

355 ccdGraySubCcdTriangular = pexConfig.Field( 

356 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

357 "suppress high-order cross terms?"), 

358 dtype=bool, 

359 default=True, 

360 ) 

361 ccdGrayFocalPlaneDict = pexConfig.DictField( 

362 doc=("Per-band specification on whether to compute focal-plane residual " 

363 "('ccd gray') corrections."), 

364 keytype=str, 

365 itemtype=bool, 

366 default={}, 

367 ) 

368 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

369 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

370 "gray corrections. If there are fewer good CCDs then the gray " 

371 "correction is computed per-ccd."), 

372 dtype=int, 

373 default=1, 

374 ) 

375 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

376 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

377 dtype=int, 

378 default=3, 

379 ) 

380 cycleNumber = pexConfig.Field( 

381 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

382 "and stage of outlier rejection. See cookbook for details."), 

383 dtype=int, 

384 default=None, 

385 ) 

386 isFinalCycle = pexConfig.Field( 

387 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

388 "selection of stars and photometric exposures, and will output zeropoints " 

389 "and standard stars for use in fgcmOutputProducts"), 

390 dtype=bool, 

391 default=False, 

392 ) 

393 maxIterBeforeFinalCycle = pexConfig.Field( 

394 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

395 "will always be 0 in the final cycle for cleanup and final selection."), 

396 dtype=int, 

397 default=50, 

398 ) 

399 deltaMagBkgOffsetPercentile = pexConfig.Field( 

400 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

401 "offset from local background subtraction."), 

402 dtype=float, 

403 default=0.25, 

404 ) 

405 deltaMagBkgPerCcd = pexConfig.Field( 

406 doc=("Compute net offset from local background subtraction per-ccd? " 

407 "Otherwise, use computation per visit."), 

408 dtype=bool, 

409 default=False, 

410 ) 

411 utBoundary = pexConfig.Field( 

412 doc="Boundary (in UTC) from day-to-day", 

413 dtype=float, 

414 default=None, 

415 ) 

416 washMjds = pexConfig.ListField( 

417 doc="Mirror wash MJDs", 

418 dtype=float, 

419 default=(0.0,), 

420 ) 

421 epochMjds = pexConfig.ListField( 

422 doc="Epoch boundaries in MJD", 

423 dtype=float, 

424 default=(0.0,), 

425 ) 

426 minObsPerBand = pexConfig.Field( 

427 doc="Minimum good observations per band", 

428 dtype=int, 

429 default=2, 

430 ) 

431 # TODO: When DM-16511 is done, it will be possible to get the 

432 # telescope latitude directly from the camera. 

433 latitude = pexConfig.Field( 

434 doc="Observatory latitude", 

435 dtype=float, 

436 default=None, 

437 ) 

438 brightObsGrayMax = pexConfig.Field( 

439 doc="Maximum gray extinction to be considered bright observation", 

440 dtype=float, 

441 default=0.15, 

442 ) 

443 minStarPerCcd = pexConfig.Field( 

444 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

445 "CCDs with fewer stars will have their calibration estimated from other " 

446 "CCDs in the same visit, with zeropoint error increased accordingly."), 

447 dtype=int, 

448 default=5, 

449 ) 

450 minCcdPerExp = pexConfig.Field( 

451 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

452 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

453 dtype=int, 

454 default=5, 

455 ) 

456 maxCcdGrayErr = pexConfig.Field( 

457 doc="Maximum error on CCD gray offset to be considered photometric", 

458 dtype=float, 

459 default=0.05, 

460 ) 

461 minStarPerExp = pexConfig.Field( 

462 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

463 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

464 dtype=int, 

465 default=600, 

466 ) 

467 minExpPerNight = pexConfig.Field( 

468 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

469 dtype=int, 

470 default=10, 

471 ) 

472 expGrayInitialCut = pexConfig.Field( 

473 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

474 "observations."), 

475 dtype=float, 

476 default=-0.25, 

477 ) 

478 expGrayPhotometricCut = pexConfig.ListField( 

479 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. " 

480 "Must be same length as config.bands, and matched band-by-band."), 

481 dtype=float, 

482 default=(0.0,), 

483 optional=True, 

484 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

485 "It will be removed after v20. Use expGrayPhotometricCutDict instead."), 

486 ) 

487 expGrayPhotometricCutDict = pexConfig.DictField( 

488 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

489 "('gray term') for a visit to be considered photometric. Must have one " 

490 "entry per band. Broad-band filters should be -0.05."), 

491 keytype=str, 

492 itemtype=float, 

493 default={}, 

494 ) 

495 expGrayHighCut = pexConfig.ListField( 

496 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. " 

497 "Must be same length as config.bands, and matched band-by-band."), 

498 dtype=float, 

499 default=(0.0,), 

500 optional=True, 

501 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

502 "It will be removed after v20. Use expGrayHighCutDict instead."), 

503 ) 

504 expGrayHighCutDict = pexConfig.DictField( 

505 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

506 "('gray term') for a visit to be considered photometric. Must have one " 

507 "entry per band. Broad-band filters should be 0.2."), 

508 keytype=str, 

509 itemtype=float, 

510 default={}, 

511 ) 

512 expGrayRecoverCut = pexConfig.Field( 

513 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

514 "Visits with more gray extinction will only get CCD zeropoints if there are " 

515 "sufficient star observations (minStarPerCcd) on that CCD."), 

516 dtype=float, 

517 default=-1.0, 

518 ) 

519 expVarGrayPhotometricCut = pexConfig.Field( 

520 doc="Maximum exposure variance to be considered possibly photometric", 

521 dtype=float, 

522 default=0.0005, 

523 optional=True, 

524 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

525 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."), 

526 ) 

527 expVarGrayPhotometricCutDict = pexConfig.DictField( 

528 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

529 "photometric. Must have one entry per band. Broad-band filters should be " 

530 "0.0005."), 

531 keytype=str, 

532 itemtype=float, 

533 default={}, 

534 ) 

535 expGrayErrRecoverCut = pexConfig.Field( 

536 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

537 "Visits with more gray variance will only get CCD zeropoints if there are " 

538 "sufficient star observations (minStarPerCcd) on that CCD."), 

539 dtype=float, 

540 default=0.05, 

541 ) 

542 aperCorrFitNBins = pexConfig.Field( 

543 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

544 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

545 "used if available."), 

546 dtype=int, 

547 default=10, 

548 ) 

549 aperCorrInputSlopes = pexConfig.ListField( 

550 doc=("Aperture correction input slope parameters. These are used on the first " 

551 "fit iteration, and aperture correction parameters will be updated from " 

552 "the data if config.aperCorrFitNBins > 0. It is recommended to set this" 

553 "when there is insufficient data to fit the parameters (e.g. tract mode). " 

554 "If set, must be same length as config.bands, and matched band-by-band."), 

555 dtype=float, 

556 default=[], 

557 optional=True, 

558 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

559 "It will be removed after v20. Use aperCorrInputSlopeDict instead."), 

560 ) 

561 aperCorrInputSlopeDict = pexConfig.DictField( 

562 doc=("Per-band specification of aperture correction input slope parameters. These " 

563 "are used on the first fit iteration, and aperture correction parameters will " 

564 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

565 "to set this when there is insufficient data to fit the parameters (e.g. " 

566 "tract mode)."), 

567 keytype=str, 

568 itemtype=float, 

569 default={}, 

570 ) 

571 sedFudgeFactors = pexConfig.ListField( 

572 doc=("Fudge factors for computing linear SED from colors. Must be same length as " 

573 "config.bands, and matched band-by-band."), 

574 dtype=float, 

575 default=(0,), 

576 optional=True, 

577 deprecated=("This field has been deprecated and will be removed after v20. " 

578 "Please use sedSlopeTermMap and sedSlopeMap."), 

579 ) 

580 sedboundaryterms = pexConfig.ConfigField( 

581 doc="Mapping from bands to SED boundary term names used is sedterms.", 

582 dtype=SedboundarytermDict, 

583 ) 

584 sedterms = pexConfig.ConfigField( 

585 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

586 dtype=SedtermDict, 

587 ) 

588 sigFgcmMaxErr = pexConfig.Field( 

589 doc="Maximum mag error for fitting sigma_FGCM", 

590 dtype=float, 

591 default=0.01, 

592 ) 

593 sigFgcmMaxEGray = pexConfig.ListField( 

594 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. " 

595 "May be 1 element (same for all bands) or the same length as config.bands."), 

596 dtype=float, 

597 default=(0.05,), 

598 optional=True, 

599 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

600 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."), 

601 ) 

602 sigFgcmMaxEGrayDict = pexConfig.DictField( 

603 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

604 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

605 "should be 0.05."), 

606 keytype=str, 

607 itemtype=float, 

608 default={}, 

609 ) 

610 ccdGrayMaxStarErr = pexConfig.Field( 

611 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

612 "computation"), 

613 dtype=float, 

614 default=0.10, 

615 ) 

616 approxThroughput = pexConfig.ListField( 

617 doc=("Approximate overall throughput at start of calibration observations. " 

618 "May be 1 element (same for all bands) or the same length as config.bands."), 

619 dtype=float, 

620 default=(1.0, ), 

621 optional=True, 

622 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

623 "It will be removed after v20. Use approxThroughputDict instead."), 

624 ) 

625 approxThroughputDict = pexConfig.DictField( 

626 doc=("Per-band specification of the approximate overall throughput at the start of " 

627 "calibration observations. Must have one entry per band. Typically should " 

628 "be 1.0."), 

629 keytype=str, 

630 itemtype=float, 

631 default={}, 

632 ) 

633 sigmaCalRange = pexConfig.ListField( 

634 doc="Allowed range for systematic error floor estimation", 

635 dtype=float, 

636 default=(0.001, 0.003), 

637 ) 

638 sigmaCalFitPercentile = pexConfig.ListField( 

639 doc="Magnitude percentile range to fit systematic error floor", 

640 dtype=float, 

641 default=(0.05, 0.15), 

642 ) 

643 sigmaCalPlotPercentile = pexConfig.ListField( 

644 doc="Magnitude percentile range to plot systematic error floor", 

645 dtype=float, 

646 default=(0.05, 0.95), 

647 ) 

648 sigma0Phot = pexConfig.Field( 

649 doc="Systematic error floor for all zeropoints", 

650 dtype=float, 

651 default=0.003, 

652 ) 

653 mapLongitudeRef = pexConfig.Field( 

654 doc="Reference longitude for plotting maps", 

655 dtype=float, 

656 default=0.0, 

657 ) 

658 mapNSide = pexConfig.Field( 

659 doc="Healpix nside for plotting maps", 

660 dtype=int, 

661 default=256, 

662 ) 

663 outfileBase = pexConfig.Field( 

664 doc="Filename start for plot output files", 

665 dtype=str, 

666 default=None, 

667 ) 

668 starColorCuts = pexConfig.ListField( 

669 doc="Encoded star-color cuts (to be cleaned up)", 

670 dtype=str, 

671 default=("NO_DATA",), 

672 ) 

673 colorSplitIndices = pexConfig.ListField( 

674 doc="Band indices to use to split stars by color", 

675 dtype=int, 

676 default=None, 

677 optional=True, 

678 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

679 "It will be removed after v20. Use colorSplitBands instead."), 

680 ) 

681 colorSplitBands = pexConfig.ListField( 

682 doc="Band names to use to split stars by color. Must have 2 entries.", 

683 dtype=str, 

684 length=2, 

685 default=('g', 'i'), 

686 ) 

687 modelMagErrors = pexConfig.Field( 

688 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

689 dtype=bool, 

690 default=True, 

691 ) 

692 useQuadraticPwv = pexConfig.Field( 

693 doc="Model PWV with a quadratic term for variation through the night?", 

694 dtype=bool, 

695 default=False, 

696 ) 

697 instrumentParsPerBand = pexConfig.Field( 

698 doc=("Model instrumental parameters per band? " 

699 "Otherwise, instrumental parameters (QE changes with time) are " 

700 "shared among all bands."), 

701 dtype=bool, 

702 default=False, 

703 ) 

704 instrumentSlopeMinDeltaT = pexConfig.Field( 

705 doc=("Minimum time change (in days) between observations to use in constraining " 

706 "instrument slope."), 

707 dtype=float, 

708 default=20.0, 

709 ) 

710 fitMirrorChromaticity = pexConfig.Field( 

711 doc="Fit (intraband) mirror chromatic term?", 

712 dtype=bool, 

713 default=False, 

714 ) 

715 coatingMjds = pexConfig.ListField( 

716 doc="Mirror coating dates in MJD", 

717 dtype=float, 

718 default=(0.0,), 

719 ) 

720 outputStandardsBeforeFinalCycle = pexConfig.Field( 

721 doc="Output standard stars prior to final cycle? Used in debugging.", 

722 dtype=bool, 

723 default=False, 

724 ) 

725 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

726 doc="Output standard stars prior to final cycle? Used in debugging.", 

727 dtype=bool, 

728 default=False, 

729 ) 

730 useRepeatabilityForExpGrayCuts = pexConfig.ListField( 

731 doc=("Use star repeatability (instead of exposures) for computing photometric " 

732 "cuts? Recommended for tract mode or bands with few exposures. " 

733 "May be 1 element (same for all bands) or the same length as config.bands."), 

734 dtype=bool, 

735 default=(False,), 

736 optional=True, 

737 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

738 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."), 

739 ) 

740 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

741 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

742 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

743 keytype=str, 

744 itemtype=bool, 

745 default={}, 

746 ) 

747 autoPhotometricCutNSig = pexConfig.Field( 

748 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

749 "Cut is based on exposure gray width (per band), unless " 

750 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

751 "repeatability is used (also per band)."), 

752 dtype=float, 

753 default=3.0, 

754 ) 

755 autoHighCutNSig = pexConfig.Field( 

756 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

757 "Cut is based on exposure gray width (per band), unless " 

758 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

759 "repeatability is used (also per band)."), 

760 dtype=float, 

761 default=4.0, 

762 ) 

763 quietMode = pexConfig.Field( 

764 doc="Be less verbose with logging.", 

765 dtype=bool, 

766 default=False, 

767 ) 

768 randomSeed = pexConfig.Field( 

769 doc="Random seed for fgcm for consistency in tests.", 

770 dtype=int, 

771 default=None, 

772 optional=True, 

773 ) 

774 

775 def validate(self): 

776 super().validate() 

777 

778 if self.connections.previousCycleNumber != str(self.cycleNumber - 1): 

779 msg = "cycleNumber in template must be connections.previousCycleNumber + 1" 

780 raise RuntimeError(msg) 

781 if self.connections.cycleNumber != str(self.cycleNumber): 

782 msg = "cycleNumber in template must be equal to connections.cycleNumber" 

783 raise RuntimeError(msg) 

784 

785 for band in self.fitBands: 

786 if band not in self.bands: 

787 msg = 'fitBand %s not in bands' % (band) 

788 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

789 for band in self.requiredBands: 

790 if band not in self.bands: 

791 msg = 'requiredBand %s not in bands' % (band) 

792 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

793 for band in self.colorSplitBands: 

794 if band not in self.bands: 

795 msg = 'colorSplitBand %s not in bands' % (band) 

796 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

797 for band in self.bands: 

798 if band not in self.superStarSubCcdDict: 

799 msg = 'band %s not in superStarSubCcdDict' % (band) 

800 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

801 self, msg) 

802 if band not in self.ccdGraySubCcdDict: 

803 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

804 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

805 self, msg) 

806 if band not in self.expGrayPhotometricCutDict: 

807 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

808 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

809 self, msg) 

810 if band not in self.expGrayHighCutDict: 

811 msg = 'band %s not in expGrayHighCutDict' % (band) 

812 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

813 self, msg) 

814 if band not in self.expVarGrayPhotometricCutDict: 

815 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

816 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

817 self, msg) 

818 if band not in self.sigFgcmMaxEGrayDict: 

819 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

820 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

821 self, msg) 

822 if band not in self.approxThroughputDict: 

823 msg = 'band %s not in approxThroughputDict' % (band) 

824 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

825 self, msg) 

826 if band not in self.useRepeatabilityForExpGrayCutsDict: 

827 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

828 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

829 self, msg) 

830 

831 

832class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner): 

833 """Subclass of TaskRunner for fgcmFitCycleTask 

834 

835 fgcmFitCycleTask.run() takes one argument, the butler, and uses 

836 stars and visits previously extracted from dataRefs by 

837 fgcmBuildStars. 

838 This Runner does not perform any dataRef parallelization, but the FGCM 

839 code called by the Task uses python multiprocessing (see the "ncores" 

840 config option). 

841 """ 

842 

843 @staticmethod 

844 def getTargetList(parsedCmd): 

845 """ 

846 Return a list with one element, the butler. 

847 """ 

848 return [parsedCmd.butler] 

849 

850 def __call__(self, butler): 

851 """ 

852 Parameters 

853 ---------- 

854 butler: `lsst.daf.persistence.Butler` 

855 

856 Returns 

857 ------- 

858 exitStatus: `list` with `pipeBase.Struct` 

859 exitStatus (0: success; 1: failure) 

860 """ 

861 

862 task = self.TaskClass(config=self.config, log=self.log) 

863 

864 exitStatus = 0 

865 if self.doRaise: 

866 task.runDataRef(butler) 

867 else: 

868 try: 

869 task.runDataRef(butler) 

870 except Exception as e: 

871 exitStatus = 1 

872 task.log.fatal("Failed: %s" % e) 

873 if not isinstance(e, pipeBase.TaskError): 

874 traceback.print_exc(file=sys.stderr) 

875 

876 task.writeMetadata(butler) 

877 

878 # The task does not return any results: 

879 return [pipeBase.Struct(exitStatus=exitStatus)] 

880 

881 def run(self, parsedCmd): 

882 """ 

883 Run the task, with no multiprocessing 

884 

885 Parameters 

886 ---------- 

887 parsedCmd: ArgumentParser parsed command line 

888 """ 

889 

890 resultList = [] 

891 

892 if self.precall(parsedCmd): 

893 targetList = self.getTargetList(parsedCmd) 

894 # make sure that we only get 1 

895 resultList = self(targetList[0]) 

896 

897 return resultList 

898 

899 

900class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

901 """ 

902 Run Single fit cycle for FGCM global calibration 

903 """ 

904 

905 ConfigClass = FgcmFitCycleConfig 

906 RunnerClass = FgcmFitCycleRunner 

907 _DefaultName = "fgcmFitCycle" 

908 

909 def __init__(self, butler=None, initInputs=None, **kwargs): 

910 super().__init__(**kwargs) 

911 

912 # no saving of metadata for now 

913 def _getMetadataName(self): 

914 return None 

915 

916 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

917 camera = butlerQC.get(inputRefs.camera) 

918 

919 dataRefDict = {} 

920 

921 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

922 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

923 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations) 

924 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds) 

925 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices) 

926 if self.config.doReferenceCalibration: 

927 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars) 

928 if self.config.cycleNumber > 0: 

929 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput) 

930 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput) 

931 

932 fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict) 

933 

934 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters) 

935 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars) 

936 if self.outputZeropoints: 

937 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints) 

938 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters) 

939 if self.outputStandards: 

940 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars) 

941 

942 @pipeBase.timeMethod 

943 def runDataRef(self, butler): 

944 """ 

945 Run a single fit cycle for FGCM 

946 

947 Parameters 

948 ---------- 

949 butler: `lsst.daf.persistence.Butler` 

950 """ 

951 self._checkDatasetsExist(butler) 

952 

953 dataRefDict = {} 

954 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

955 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

956 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations') 

957 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds') 

958 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices') 

959 if self.config.doReferenceCalibration: 

960 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars') 

961 if self.config.cycleNumber > 0: 

962 lastCycle = self.config.cycleNumber - 1 

963 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars', 

964 fgcmcycle=lastCycle) 

965 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters', 

966 fgcmcycle=lastCycle) 

967 

968 camera = butler.get('camera') 

969 fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict) 

970 

971 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters', 

972 fgcmcycle=self.config.cycleNumber) 

973 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars', 

974 fgcmcycle=self.config.cycleNumber) 

975 if self.outputZeropoints: 

976 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints', 

977 fgcmcycle=self.config.cycleNumber) 

978 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters', 

979 fgcmcycle=self.config.cycleNumber) 

980 if self.outputStandards: 

981 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars', 

982 fgcmcycle=self.config.cycleNumber) 

983 

984 def writeConfig(self, butler, clobber=False, doBackup=True): 

985 """Write the configuration used for processing the data, or check that an existing 

986 one is equal to the new one if present. This is an override of the regular 

987 version from pipe_base that knows about fgcmcycle. 

988 

989 Parameters 

990 ---------- 

991 butler : `lsst.daf.persistence.Butler` 

992 Data butler used to write the config. The config is written to dataset type 

993 `CmdLineTask._getConfigName`. 

994 clobber : `bool`, optional 

995 A boolean flag that controls what happens if a config already has been saved: 

996 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 

997 - `False`: raise `TaskError` if this config does not match the existing config. 

998 doBackup : `bool`, optional 

999 Set to `True` to backup the config files if clobbering. 

1000 """ 

1001 configName = self._getConfigName() 

1002 if configName is None: 

1003 return 

1004 if clobber: 

1005 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber) 

1006 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber): 

1007 # this may be subject to a race condition; see #2789 

1008 try: 

1009 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber) 

1010 except Exception as exc: 

1011 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" % 

1012 (configName, exc)) 

1013 

1014 def logConfigMismatch(msg): 

1015 self.log.fatal("Comparing configuration: %s", msg) 

1016 

1017 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

1018 raise pipeBase.TaskError( 

1019 f"Config does not match existing task config {configName!r} on disk; tasks configurations" 

1020 " must be consistent within the same output repo (override with --clobber-config)") 

1021 else: 

1022 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber) 

1023 

1024 def _fgcmFitCycle(self, camera, dataRefDict): 

1025 """ 

1026 Run the fit cycle 

1027 

1028 Parameters 

1029 ---------- 

1030 camera : `lsst.afw.cameraGeom.Camera` 

1031 dataRefDict : `dict` 

1032 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1033 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1034 dataRef dictionary with keys: 

1035 

1036 ``"fgcmLookUpTable"`` 

1037 dataRef for the FGCM look-up table. 

1038 ``"fgcmVisitCatalog"`` 

1039 dataRef for visit summary catalog. 

1040 ``"fgcmStarObservations"`` 

1041 dataRef for star observation catalog. 

1042 ``"fgcmStarIds"`` 

1043 dataRef for star id catalog. 

1044 ``"fgcmStarIndices"`` 

1045 dataRef for star index catalog. 

1046 ``"fgcmReferenceStars"`` 

1047 dataRef for matched reference star catalog. 

1048 ``"fgcmFlaggedStars"`` 

1049 dataRef for flagged star catalog. 

1050 ``"fgcmFitParameters"`` 

1051 dataRef for fit parameter catalog. 

1052 

1053 Returns 

1054 ------- 

1055 fgcmDatasetDict : `dict` 

1056 Dictionary of datasets to persist. 

1057 """ 

1058 # Set defaults on whether to output standards and zeropoints 

1059 self.maxIter = self.config.maxIterBeforeFinalCycle 

1060 self.outputStandards = self.config.outputStandardsBeforeFinalCycle 

1061 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle 

1062 self.resetFitParameters = True 

1063 

1064 if self.config.isFinalCycle: 

1065 # This is the final fit cycle, so we do not want to reset fit 

1066 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

1067 # and we always want to output standards and zeropoints 

1068 self.maxIter = 0 

1069 self.outputStandards = True 

1070 self.outputZeropoints = True 

1071 self.resetFitParameters = False 

1072 

1073 configDict = makeConfigDict(self.config, self.log, camera, 

1074 self.maxIter, self.resetFitParameters, 

1075 self.outputZeropoints) 

1076 

1077 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1078 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap)) 

1079 del lutCat 

1080 

1081 # next we need the exposure/visit information 

1082 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

1083 fgcmExpInfo = translateVisitCatalog(visitCat) 

1084 del visitCat 

1085 

1086 # Use the first orientation. 

1087 # TODO: DM-21215 will generalize to arbitrary camera orientations 

1088 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

1089 

1090 noFitsDict = {'lutIndex': lutIndexVals, 

1091 'lutStd': lutStd, 

1092 'expInfo': fgcmExpInfo, 

1093 'ccdOffsets': ccdOffsets} 

1094 

1095 # set up the fitter object 

1096 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

1097 noFitsDict=noFitsDict, noOutput=True) 

1098 

1099 # create the parameter object 

1100 if (fgcmFitCycle.initialCycle): 

1101 # cycle = 0, initial cycle 

1102 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

1103 fgcmLut, 

1104 fgcmExpInfo) 

1105 else: 

1106 inParInfo, inParams, inSuperStar = self._loadParameters(dataRefDict['fgcmFitParameters']) 

1107 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

1108 fgcmExpInfo, 

1109 inParInfo, 

1110 inParams, 

1111 inSuperStar) 

1112 

1113 # set up the stars... 

1114 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

1115 

1116 starObs = dataRefDict['fgcmStarObservations'].get() 

1117 starIds = dataRefDict['fgcmStarIds'].get() 

1118 starIndices = dataRefDict['fgcmStarIndices'].get() 

1119 

1120 # grab the flagged stars if available 

1121 if 'fgcmFlaggedStars' in dataRefDict: 

1122 flaggedStars = dataRefDict['fgcmFlaggedStars'].get() 

1123 flagId = flaggedStars['objId'][:] 

1124 flagFlag = flaggedStars['objFlag'][:] 

1125 else: 

1126 flaggedStars = None 

1127 flagId = None 

1128 flagFlag = None 

1129 

1130 if self.config.doReferenceCalibration: 

1131 refStars = dataRefDict['fgcmReferenceStars'].get() 

1132 

1133 refMag, refMagErr = extractReferenceMags(refStars, 

1134 self.config.bands, 

1135 self.config.filterMap) 

1136 refId = refStars['fgcm_id'][:] 

1137 else: 

1138 refStars = None 

1139 refId = None 

1140 refMag = None 

1141 refMagErr = None 

1142 

1143 # match star observations to visits 

1144 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

1145 # actually be transferred into fgcm using the indexing below. 

1146 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

1147 

1148 # The fgcmStars.loadStars method will copy all the star information into 

1149 # special shared memory objects that will not blow up the memory usage when 

1150 # used with python multiprocessing. Once all the numbers are copied, 

1151 # it is necessary to release all references to the objects that previously 

1152 # stored the data to ensure that the garbage collector can clear the memory, 

1153 # and ensure that this memory is not copied when multiprocessing kicks in. 

1154 

1155 # We determine the conversion from the native units (typically radians) to 

1156 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

1157 # numpy arrays rather than Angles, which would we approximately 600x slower. 

1158 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

1159 

1160 fgcmStars.loadStars(fgcmPars, 

1161 starObs['visit'][starIndices['obsIndex']], 

1162 starObs['ccd'][starIndices['obsIndex']], 

1163 starObs['ra'][starIndices['obsIndex']] * conv, 

1164 starObs['dec'][starIndices['obsIndex']] * conv, 

1165 starObs['instMag'][starIndices['obsIndex']], 

1166 starObs['instMagErr'][starIndices['obsIndex']], 

1167 fgcmExpInfo['FILTERNAME'][visitIndex], 

1168 starIds['fgcm_id'][:], 

1169 starIds['ra'][:], 

1170 starIds['dec'][:], 

1171 starIds['obsArrIndex'][:], 

1172 starIds['nObs'][:], 

1173 obsX=starObs['x'][starIndices['obsIndex']], 

1174 obsY=starObs['y'][starIndices['obsIndex']], 

1175 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']], 

1176 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

1177 refID=refId, 

1178 refMag=refMag, 

1179 refMagErr=refMagErr, 

1180 flagID=flagId, 

1181 flagFlag=flagFlag, 

1182 computeNobs=True) 

1183 

1184 # Release all references to temporary objects holding star data (see above) 

1185 del starObs 

1186 del starIds 

1187 del starIndices 

1188 del flagId 

1189 del flagFlag 

1190 del flaggedStars 

1191 del refStars 

1192 del refId 

1193 del refMag 

1194 del refMagErr 

1195 

1196 # and set the bits in the cycle object 

1197 fgcmFitCycle.setLUT(fgcmLut) 

1198 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

1199 fgcmFitCycle.setPars(fgcmPars) 

1200 

1201 # finish the setup 

1202 fgcmFitCycle.finishSetup() 

1203 

1204 # and run 

1205 fgcmFitCycle.run() 

1206 

1207 ################## 

1208 # Persistance 

1209 ################## 

1210 

1211 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle) 

1212 

1213 # Output the config for the next cycle 

1214 # We need to make a copy since the input one has been frozen 

1215 

1216 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

1217 i, b in enumerate(self.config.bands)} 

1218 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

1219 i, band in enumerate(self.config.bands)} 

1220 

1221 outConfig = copy.copy(self.config) 

1222 outConfig.update(cycleNumber=(self.config.cycleNumber + 1), 

1223 precomputeSuperStarInitialCycle=False, 

1224 freezeStdAtmosphere=False, 

1225 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

1226 expGrayHighCutDict=updatedHighCutDict) 

1227 

1228 outConfig.connections.update(previousCycleNumber=str(self.config.cycleNumber), 

1229 cycleNumber=str(self.config.cycleNumber + 1)) 

1230 

1231 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

1232 outConfig.cycleNumber) 

1233 outConfig.save(configFileName) 

1234 

1235 if self.config.isFinalCycle == 1: 

1236 # We are done, ready to output products 

1237 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

1238 else: 

1239 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

1240 self.log.info("Be sure to look at:") 

1241 self.log.info(" config.expGrayPhotometricCut") 

1242 self.log.info(" config.expGrayHighCut") 

1243 self.log.info("If you are satisfied with the fit, please set:") 

1244 self.log.info(" config.isFinalCycle = True") 

1245 

1246 return fgcmDatasetDict 

1247 

1248 def _checkDatasetsExist(self, butler): 

1249 """ 

1250 Check if necessary datasets exist to run fgcmFitCycle 

1251 

1252 Parameters 

1253 ---------- 

1254 butler: `lsst.daf.persistence.Butler` 

1255 

1256 Raises 

1257 ------ 

1258 RuntimeError 

1259 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 

1260 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 

1261 If cycleNumber > 0, then also checks for fgcmFitParameters, 

1262 fgcmFlaggedStars. 

1263 """ 

1264 

1265 if not butler.datasetExists('fgcmVisitCatalog'): 

1266 raise RuntimeError("Could not find fgcmVisitCatalog in repo!") 

1267 if not butler.datasetExists('fgcmStarObservations'): 

1268 raise RuntimeError("Could not find fgcmStarObservations in repo!") 

1269 if not butler.datasetExists('fgcmStarIds'): 

1270 raise RuntimeError("Could not find fgcmStarIds in repo!") 

1271 if not butler.datasetExists('fgcmStarIndices'): 

1272 raise RuntimeError("Could not find fgcmStarIndices in repo!") 

1273 if not butler.datasetExists('fgcmLookUpTable'): 

1274 raise RuntimeError("Could not find fgcmLookUpTable in repo!") 

1275 

1276 # Need additional datasets if we are not the initial cycle 

1277 if (self.config.cycleNumber > 0): 

1278 if not butler.datasetExists('fgcmFitParameters', 

1279 fgcmcycle=self.config.cycleNumber-1): 

1280 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" % 

1281 (self.config.cycleNumber-1)) 

1282 if not butler.datasetExists('fgcmFlaggedStars', 

1283 fgcmcycle=self.config.cycleNumber-1): 

1284 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" % 

1285 (self.config.cycleNumber-1)) 

1286 

1287 # And additional dataset if we want reference calibration 

1288 if self.config.doReferenceCalibration: 

1289 if not butler.datasetExists('fgcmReferenceStars'): 

1290 raise RuntimeError("Could not find fgcmReferenceStars in repo, and " 

1291 "doReferenceCalibration is True.") 

1292 

1293 def _loadParameters(self, parDataRef): 

1294 """ 

1295 Load FGCM parameters from a previous fit cycle 

1296 

1297 Parameters 

1298 ---------- 

1299 parDataRef : `lsst.daf.persistence.ButlerDataRef` or `lsst.daf.butler.DeferredDatasetHandle` 

1300 dataRef for previous fit parameter catalog. 

1301 

1302 Returns 

1303 ------- 

1304 inParInfo: `numpy.ndarray` 

1305 Numpy array parameter information formatted for input to fgcm 

1306 inParameters: `numpy.ndarray` 

1307 Numpy array parameter values formatted for input to fgcm 

1308 inSuperStar: `numpy.array` 

1309 Superstar flat formatted for input to fgcm 

1310 """ 

1311 # note that we already checked that this is available 

1312 parCat = parDataRef.get() 

1313 

1314 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1315 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1316 

1317 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1318 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1319 (parLutFilterNames.size, )), 

1320 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1321 ('LNTAUUNIT', 'f8'), 

1322 ('LNTAUSLOPEUNIT', 'f8'), 

1323 ('ALPHAUNIT', 'f8'), 

1324 ('LNPWVUNIT', 'f8'), 

1325 ('LNPWVSLOPEUNIT', 'f8'), 

1326 ('LNPWVQUADRATICUNIT', 'f8'), 

1327 ('LNPWVGLOBALUNIT', 'f8'), 

1328 ('O3UNIT', 'f8'), 

1329 ('QESYSUNIT', 'f8'), 

1330 ('FILTEROFFSETUNIT', 'f8'), 

1331 ('HASEXTERNALPWV', 'i2'), 

1332 ('HASEXTERNALTAU', 'i2')]) 

1333 inParInfo['NCCD'] = parCat['nCcd'] 

1334 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1335 inParInfo['FITBANDS'][:] = parFitBands 

1336 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1337 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1338 

1339 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1340 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1341 ('PARLNTAUINTERCEPT', 'f8', 

1342 (parCat['parLnTauIntercept'].size, )), 

1343 ('PARLNTAUSLOPE', 'f8', 

1344 (parCat['parLnTauSlope'].size, )), 

1345 ('PARLNPWVINTERCEPT', 'f8', 

1346 (parCat['parLnPwvIntercept'].size, )), 

1347 ('PARLNPWVSLOPE', 'f8', 

1348 (parCat['parLnPwvSlope'].size, )), 

1349 ('PARLNPWVQUADRATIC', 'f8', 

1350 (parCat['parLnPwvQuadratic'].size, )), 

1351 ('PARQESYSINTERCEPT', 'f8', 

1352 (parCat['parQeSysIntercept'].size, )), 

1353 ('COMPQESYSSLOPE', 'f8', 

1354 (parCat['compQeSysSlope'].size, )), 

1355 ('PARFILTEROFFSET', 'f8', 

1356 (parCat['parFilterOffset'].size, )), 

1357 ('PARFILTEROFFSETFITFLAG', 'i2', 

1358 (parCat['parFilterOffsetFitFlag'].size, )), 

1359 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1360 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1361 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1362 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1363 ('COMPABSTHROUGHPUT', 'f8', 

1364 (parCat['compAbsThroughput'].size, )), 

1365 ('COMPREFOFFSET', 'f8', 

1366 (parCat['compRefOffset'].size, )), 

1367 ('COMPREFSIGMA', 'f8', 

1368 (parCat['compRefSigma'].size, )), 

1369 ('COMPMIRRORCHROMATICITY', 'f8', 

1370 (parCat['compMirrorChromaticity'].size, )), 

1371 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1372 (parCat['mirrorChromaticityPivot'].size, )), 

1373 ('COMPMEDIANSEDSLOPE', 'f8', 

1374 (parCat['compMedianSedSlope'].size, )), 

1375 ('COMPAPERCORRPIVOT', 'f8', 

1376 (parCat['compAperCorrPivot'].size, )), 

1377 ('COMPAPERCORRSLOPE', 'f8', 

1378 (parCat['compAperCorrSlope'].size, )), 

1379 ('COMPAPERCORRSLOPEERR', 'f8', 

1380 (parCat['compAperCorrSlopeErr'].size, )), 

1381 ('COMPAPERCORRRANGE', 'f8', 

1382 (parCat['compAperCorrRange'].size, )), 

1383 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1384 (parCat['compModelErrExptimePivot'].size, )), 

1385 ('COMPMODELERRFWHMPIVOT', 'f8', 

1386 (parCat['compModelErrFwhmPivot'].size, )), 

1387 ('COMPMODELERRSKYPIVOT', 'f8', 

1388 (parCat['compModelErrSkyPivot'].size, )), 

1389 ('COMPMODELERRPARS', 'f8', 

1390 (parCat['compModelErrPars'].size, )), 

1391 ('COMPEXPGRAY', 'f8', 

1392 (parCat['compExpGray'].size, )), 

1393 ('COMPVARGRAY', 'f8', 

1394 (parCat['compVarGray'].size, )), 

1395 ('COMPEXPDELTAMAGBKG', 'f8', 

1396 (parCat['compExpDeltaMagBkg'].size, )), 

1397 ('COMPNGOODSTARPEREXP', 'i4', 

1398 (parCat['compNGoodStarPerExp'].size, )), 

1399 ('COMPSIGFGCM', 'f8', 

1400 (parCat['compSigFgcm'].size, )), 

1401 ('COMPSIGMACAL', 'f8', 

1402 (parCat['compSigmaCal'].size, )), 

1403 ('COMPRETRIEVEDLNPWV', 'f8', 

1404 (parCat['compRetrievedLnPwv'].size, )), 

1405 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1406 (parCat['compRetrievedLnPwvRaw'].size, )), 

1407 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1408 (parCat['compRetrievedLnPwvFlag'].size, )), 

1409 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1410 (parCat['compRetrievedTauNight'].size, ))]) 

1411 

1412 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1413 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1414 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1415 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1416 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1417 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1418 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1419 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1420 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1421 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1422 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1423 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1424 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1425 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1426 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1427 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1428 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1429 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1430 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1431 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :] 

1432 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1433 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1434 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1435 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1436 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1437 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1438 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1439 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1440 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1441 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1442 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :] 

1443 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1444 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1445 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1446 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1447 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1448 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1449 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1450 

1451 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1452 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1453 

1454 return (inParInfo, inParams, inSuperStar) 

1455 

1456 def _makeFgcmOutputDatasets(self, fgcmFitCycle): 

1457 """ 

1458 Persist FGCM datasets through the butler. 

1459 

1460 Parameters 

1461 ---------- 

1462 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1463 Fgcm Fit cycle object 

1464 """ 

1465 fgcmDatasetDict = {} 

1466 

1467 # Save the parameters 

1468 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1469 

1470 parSchema = afwTable.Schema() 

1471 

1472 comma = ',' 

1473 lutFilterNameString = comma.join([n.decode('utf-8') 

1474 for n in parInfo['LUTFILTERNAMES'][0]]) 

1475 fitBandString = comma.join([n.decode('utf-8') 

1476 for n in parInfo['FITBANDS'][0]]) 

1477 

1478 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1479 lutFilterNameString, fitBandString) 

1480 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1481 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1482 lutFilterNameString, fitBandString) 

1483 

1484 fgcmDatasetDict['fgcmFitParameters'] = parCat 

1485 

1486 # Save the indices of the flagged stars 

1487 # (stars that have been (a) reserved from the fit for testing and 

1488 # (b) bad stars that have failed quality checks.) 

1489 flagStarSchema = self._makeFlagStarSchema() 

1490 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1491 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1492 

1493 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat 

1494 

1495 # Save the zeropoint information and atmospheres only if desired 

1496 if self.outputZeropoints: 

1497 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1498 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1499 

1500 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1501 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1502 

1503 fgcmDatasetDict['fgcmZeropoints'] = zptCat 

1504 

1505 # Save atmosphere values 

1506 # These are generated by the same code that generates zeropoints 

1507 atmSchema = makeAtmSchema() 

1508 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1509 

1510 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat 

1511 

1512 # Save the standard stars (if configured) 

1513 if self.outputStandards: 

1514 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1515 stdSchema = makeStdSchema(len(goodBands)) 

1516 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1517 

1518 fgcmDatasetDict['fgcmStandardStars'] = stdCat 

1519 

1520 return fgcmDatasetDict 

1521 

1522 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1523 lutFilterNameString, fitBandString): 

1524 """ 

1525 Make the parameter persistence schema 

1526 

1527 Parameters 

1528 ---------- 

1529 parInfo: `numpy.ndarray` 

1530 Parameter information returned by fgcm 

1531 pars: `numpy.ndarray` 

1532 Parameter values returned by fgcm 

1533 parSuperStarFlat: `numpy.array` 

1534 Superstar flat values returned by fgcm 

1535 lutFilterNameString: `str` 

1536 Combined string of all the lutFilterNames 

1537 fitBandString: `str` 

1538 Combined string of all the fitBands 

1539 

1540 Returns 

1541 ------- 

1542 parSchema: `afwTable.schema` 

1543 """ 

1544 

1545 parSchema = afwTable.Schema() 

1546 

1547 # parameter info section 

1548 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1549 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1550 size=len(lutFilterNameString)) 

1551 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1552 size=len(fitBandString)) 

1553 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1554 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1555 doc='Step units for ln(AOD) slope') 

1556 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1557 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1558 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1559 doc='Step units for ln(pwv) slope') 

1560 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1561 doc='Step units for ln(pwv) quadratic term') 

1562 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1563 doc='Step units for global ln(pwv) parameters') 

1564 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1565 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1566 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1567 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1568 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1569 

1570 # parameter section 

1571 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1572 size=pars['PARALPHA'].size) 

1573 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1574 size=pars['PARO3'].size) 

1575 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1576 doc='ln(Tau) intercept parameter vector', 

1577 size=pars['PARLNTAUINTERCEPT'].size) 

1578 parSchema.addField('parLnTauSlope', type='ArrayD', 

1579 doc='ln(Tau) slope parameter vector', 

1580 size=pars['PARLNTAUSLOPE'].size) 

1581 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1582 size=pars['PARLNPWVINTERCEPT'].size) 

1583 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1584 size=pars['PARLNPWVSLOPE'].size) 

1585 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1586 size=pars['PARLNPWVQUADRATIC'].size) 

1587 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1588 size=pars['PARQESYSINTERCEPT'].size) 

1589 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1590 size=pars[0]['COMPQESYSSLOPE'].size) 

1591 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1592 size=pars['PARFILTEROFFSET'].size) 

1593 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1594 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1595 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1596 doc='Global scale for retrieved ln(pwv)') 

1597 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1598 doc='Global offset for retrieved ln(pwv)') 

1599 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1600 doc='Nightly offset for retrieved ln(pwv)', 

1601 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1602 parSchema.addField('compAbsThroughput', type='ArrayD', 

1603 doc='Absolute throughput (relative to transmission curves)', 

1604 size=pars['COMPABSTHROUGHPUT'].size) 

1605 parSchema.addField('compRefOffset', type='ArrayD', 

1606 doc='Offset between reference stars and calibrated stars', 

1607 size=pars['COMPREFOFFSET'].size) 

1608 parSchema.addField('compRefSigma', type='ArrayD', 

1609 doc='Width of reference star/calibrated star distribution', 

1610 size=pars['COMPREFSIGMA'].size) 

1611 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1612 doc='Computed mirror chromaticity terms', 

1613 size=pars['COMPMIRRORCHROMATICITY'].size) 

1614 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1615 doc='Mirror chromaticity pivot mjd', 

1616 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1617 parSchema.addField('compMedianSedSlope', type='ArrayD', 

1618 doc='Computed median SED slope (per band)', 

1619 size=pars['COMPMEDIANSEDSLOPE'].size) 

1620 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1621 size=pars['COMPAPERCORRPIVOT'].size) 

1622 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1623 size=pars['COMPAPERCORRSLOPE'].size) 

1624 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1625 size=pars['COMPAPERCORRSLOPEERR'].size) 

1626 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1627 size=pars['COMPAPERCORRRANGE'].size) 

1628 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1629 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1630 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1631 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1632 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1633 size=pars['COMPMODELERRSKYPIVOT'].size) 

1634 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1635 size=pars['COMPMODELERRPARS'].size) 

1636 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1637 size=pars['COMPEXPGRAY'].size) 

1638 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1639 size=pars['COMPVARGRAY'].size) 

1640 parSchema.addField('compExpDeltaMagBkg', type='ArrayD', 

1641 doc='Computed exposure offset due to background', 

1642 size=pars['COMPEXPDELTAMAGBKG'].size) 

1643 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1644 doc='Computed number of good stars per exposure', 

1645 size=pars['COMPNGOODSTARPEREXP'].size) 

1646 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1647 size=pars['COMPSIGFGCM'].size) 

1648 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1649 size=pars['COMPSIGMACAL'].size) 

1650 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1651 size=pars['COMPRETRIEVEDLNPWV'].size) 

1652 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1653 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1654 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1655 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1656 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1657 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1658 # superstarflat section 

1659 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1660 size=4) 

1661 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1662 size=parSuperStarFlat.size) 

1663 

1664 return parSchema 

1665 

1666 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1667 lutFilterNameString, fitBandString): 

1668 """ 

1669 Make the FGCM parameter catalog for persistence 

1670 

1671 Parameters 

1672 ---------- 

1673 parSchema: `lsst.afw.table.Schema` 

1674 Parameter catalog schema 

1675 pars: `numpy.ndarray` 

1676 FGCM parameters to put into parCat 

1677 parSuperStarFlat: `numpy.array` 

1678 FGCM superstar flat array to put into parCat 

1679 lutFilterNameString: `str` 

1680 Combined string of all the lutFilterNames 

1681 fitBandString: `str` 

1682 Combined string of all the fitBands 

1683 

1684 Returns 

1685 ------- 

1686 parCat: `afwTable.BasicCatalog` 

1687 Atmosphere and instrumental model parameter catalog for persistence 

1688 """ 

1689 

1690 parCat = afwTable.BaseCatalog(parSchema) 

1691 parCat.reserve(1) 

1692 

1693 # The parameter catalog just has one row, with many columns for all the 

1694 # atmosphere and instrument fit parameters 

1695 rec = parCat.addNew() 

1696 

1697 # info section 

1698 rec['nCcd'] = parInfo['NCCD'] 

1699 rec['lutFilterNames'] = lutFilterNameString 

1700 rec['fitBands'] = fitBandString 

1701 # note these are not currently supported here. 

1702 rec['hasExternalPwv'] = 0 

1703 rec['hasExternalTau'] = 0 

1704 

1705 # parameter section 

1706 

1707 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1708 

1709 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1710 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1711 'parQeSysIntercept', 'compQeSysSlope', 

1712 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1713 'parFilterOffset', 'parFilterOffsetFitFlag', 

1714 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1715 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1716 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1717 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1718 'compModelErrSkyPivot', 'compModelErrPars', 

1719 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1720 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope', 

1721 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1722 'compRetrievedTauNight'] 

1723 

1724 for scalarName in scalarNames: 

1725 rec[scalarName] = pars[scalarName.upper()] 

1726 

1727 for arrName in arrNames: 

1728 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1729 

1730 # superstar section 

1731 rec['superstarSize'][:] = parSuperStarFlat.shape 

1732 rec['superstar'][:] = parSuperStarFlat.flatten() 

1733 

1734 return parCat 

1735 

1736 def _makeFlagStarSchema(self): 

1737 """ 

1738 Make the flagged-stars schema 

1739 

1740 Returns 

1741 ------- 

1742 flagStarSchema: `lsst.afw.table.Schema` 

1743 """ 

1744 

1745 flagStarSchema = afwTable.Schema() 

1746 

1747 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1748 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1749 

1750 return flagStarSchema 

1751 

1752 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1753 """ 

1754 Make the flagged star catalog for persistence 

1755 

1756 Parameters 

1757 ---------- 

1758 flagStarSchema: `lsst.afw.table.Schema` 

1759 Flagged star schema 

1760 flagStarStruct: `numpy.ndarray` 

1761 Flagged star structure from fgcm 

1762 

1763 Returns 

1764 ------- 

1765 flagStarCat: `lsst.afw.table.BaseCatalog` 

1766 Flagged star catalog for persistence 

1767 """ 

1768 

1769 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1770 flagStarCat.resize(flagStarStruct.size) 

1771 

1772 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1773 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1774 

1775 return flagStarCat