Coverage for python/lsst/fgcmcal/fgcmFitCycle.py: 23%

540 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-24 03:51 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import copy 

37 

38import numpy as np 

39 

40import lsst.pex.config as pexConfig 

41import lsst.pipe.base as pipeBase 

42from lsst.pipe.base import connectionTypes 

43import lsst.afw.table as afwTable 

44 

45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

46from .utilities import extractReferenceMags 

47from .utilities import makeZptSchema, makeZptCat 

48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

49from .sedterms import SedboundarytermDict, SedtermDict 

50from .utilities import lookupStaticCalibrations 

51from .focalPlaneProjector import FocalPlaneProjector 

52 

53import fgcm 

54 

55__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask'] 

56 

57MULTIPLE_CYCLES_MAX = 10 

58 

59 

60class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

61 dimensions=("instrument",), 

62 defaultTemplates={"previousCycleNumber": "-1", 

63 "cycleNumber": "0"}): 

64 camera = connectionTypes.PrerequisiteInput( 

65 doc="Camera instrument", 

66 name="camera", 

67 storageClass="Camera", 

68 dimensions=("instrument",), 

69 lookupFunction=lookupStaticCalibrations, 

70 isCalibration=True, 

71 ) 

72 

73 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

74 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

75 "chromatic corrections."), 

76 name="fgcmLookUpTable", 

77 storageClass="Catalog", 

78 dimensions=("instrument",), 

79 deferLoad=True, 

80 ) 

81 

82 fgcmVisitCatalog = connectionTypes.Input( 

83 doc="Catalog of visit information for fgcm", 

84 name="fgcmVisitCatalog", 

85 storageClass="Catalog", 

86 dimensions=("instrument",), 

87 deferLoad=True, 

88 ) 

89 

90 fgcmStarObservations = connectionTypes.Input( 

91 doc="Catalog of star observations for fgcm", 

92 name="fgcmStarObservations", 

93 storageClass="Catalog", 

94 dimensions=("instrument",), 

95 deferLoad=True, 

96 ) 

97 

98 fgcmStarIds = connectionTypes.Input( 

99 doc="Catalog of fgcm calibration star IDs", 

100 name="fgcmStarIds", 

101 storageClass="Catalog", 

102 dimensions=("instrument",), 

103 deferLoad=True, 

104 ) 

105 

106 fgcmStarIndices = connectionTypes.Input( 

107 doc="Catalog of fgcm calibration star indices", 

108 name="fgcmStarIndices", 

109 storageClass="Catalog", 

110 dimensions=("instrument",), 

111 deferLoad=True, 

112 ) 

113 

114 fgcmReferenceStars = connectionTypes.Input( 

115 doc="Catalog of fgcm-matched reference stars", 

116 name="fgcmReferenceStars", 

117 storageClass="Catalog", 

118 dimensions=("instrument",), 

119 deferLoad=True, 

120 ) 

121 

122 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput( 

123 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle", 

124 name="fgcmFlaggedStars{previousCycleNumber}", 

125 storageClass="Catalog", 

126 dimensions=("instrument",), 

127 deferLoad=True, 

128 ) 

129 

130 fgcmFitParametersInput = connectionTypes.PrerequisiteInput( 

131 doc="Catalog of fgcm fit parameters from previous fit cycle", 

132 name="fgcmFitParameters{previousCycleNumber}", 

133 storageClass="Catalog", 

134 dimensions=("instrument",), 

135 deferLoad=True, 

136 ) 

137 

138 fgcmFitParameters = connectionTypes.Output( 

139 doc="Catalog of fgcm fit parameters from current fit cycle", 

140 name="fgcmFitParameters{cycleNumber}", 

141 storageClass="Catalog", 

142 dimensions=("instrument",), 

143 ) 

144 

145 fgcmFlaggedStars = connectionTypes.Output( 

146 doc="Catalog of flagged stars for fgcm calibration from current fit cycle", 

147 name="fgcmFlaggedStars{cycleNumber}", 

148 storageClass="Catalog", 

149 dimensions=("instrument",), 

150 ) 

151 

152 fgcmZeropoints = connectionTypes.Output( 

153 doc="Catalog of fgcm zeropoint data from current fit cycle", 

154 name="fgcmZeropoints{cycleNumber}", 

155 storageClass="Catalog", 

156 dimensions=("instrument",), 

157 ) 

158 

159 fgcmAtmosphereParameters = connectionTypes.Output( 

160 doc="Catalog of atmospheric fit parameters from current fit cycle", 

161 name="fgcmAtmosphereParameters{cycleNumber}", 

162 storageClass="Catalog", 

163 dimensions=("instrument",), 

164 ) 

165 

166 fgcmStandardStars = connectionTypes.Output( 

167 doc="Catalog of standard star magnitudes from current fit cycle", 

168 name="fgcmStandardStars{cycleNumber}", 

169 storageClass="SimpleCatalog", 

170 dimensions=("instrument",), 

171 ) 

172 

173 # Add connections for running multiple cycles 

174 # This uses vars() to alter the class __dict__ to programmatically 

175 # write many similar outputs. 

176 for cycle in range(MULTIPLE_CYCLES_MAX): 

177 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output( 

178 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}", 

179 name=f"fgcmFitParameters{cycle}", 

180 storageClass="Catalog", 

181 dimensions=("instrument",), 

182 ) 

183 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output( 

184 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}", 

185 name=f"fgcmFlaggedStars{cycle}", 

186 storageClass="Catalog", 

187 dimensions=("instrument",), 

188 ) 

189 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output( 

190 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}", 

191 name=f"fgcmZeropoints{cycle}", 

192 storageClass="Catalog", 

193 dimensions=("instrument",), 

194 ) 

195 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output( 

196 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}", 

197 name=f"fgcmAtmosphereParameters{cycle}", 

198 storageClass="Catalog", 

199 dimensions=("instrument",), 

200 ) 

201 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output( 

202 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}", 

203 name=f"fgcmStandardStars{cycle}", 

204 storageClass="SimpleCatalog", 

205 dimensions=("instrument",), 

206 ) 

207 

208 def __init__(self, *, config=None): 

209 super().__init__(config=config) 

210 

211 if not config.doReferenceCalibration: 

212 self.inputs.remove("fgcmReferenceStars") 

213 

214 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

215 raise ValueError("cycleNumber must be of integer format") 

216 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

217 raise ValueError("previousCycleNumber must be of integer format") 

218 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

219 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

220 

221 if int(config.connections.cycleNumber) == 0: 

222 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput") 

223 self.prerequisiteInputs.remove("fgcmFitParametersInput") 

224 

225 if not self.config.doMultipleCycles: 

226 # Single-cycle run 

227 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle: 

228 self.outputs.remove("fgcmStandardStars") 

229 

230 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle: 

231 self.outputs.remove("fgcmZeropoints") 

232 self.outputs.remove("fgcmAtmosphereParameters") 

233 

234 # Remove all multiple cycle outputs 

235 for cycle in range(0, MULTIPLE_CYCLES_MAX): 

236 self.outputs.remove(f"fgcmFitParameters{cycle}") 

237 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

238 self.outputs.remove(f"fgcmZeropoints{cycle}") 

239 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

240 self.outputs.remove(f"fgcmStandardStars{cycle}") 

241 

242 else: 

243 # Multiple-cycle run 

244 # Remove single-cycle outputs 

245 self.outputs.remove("fgcmFitParameters") 

246 self.outputs.remove("fgcmFlaggedStars") 

247 self.outputs.remove("fgcmZeropoints") 

248 self.outputs.remove("fgcmAtmosphereParameters") 

249 self.outputs.remove("fgcmStandardStars") 

250 

251 # Remove outputs from cycles that are not used 

252 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1, 

253 MULTIPLE_CYCLES_MAX): 

254 self.outputs.remove(f"fgcmFitParameters{cycle}") 

255 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

256 self.outputs.remove(f"fgcmZeropoints{cycle}") 

257 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

258 self.outputs.remove(f"fgcmStandardStars{cycle}") 

259 

260 # Remove non-final-cycle outputs if necessary 

261 for cycle in range(self.config.multipleCyclesFinalCycleNumber): 

262 if not self.config.outputZeropointsBeforeFinalCycle: 

263 self.outputs.remove(f"fgcmZeropoints{cycle}") 

264 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

265 if not self.config.outputStandardsBeforeFinalCycle: 

266 self.outputs.remove(f"fgcmStandardStars{cycle}") 

267 

268 

269class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

270 pipelineConnections=FgcmFitCycleConnections): 

271 """Config for FgcmFitCycle""" 

272 

273 doMultipleCycles = pexConfig.Field( 

274 doc="Run multiple fit cycles in one task", 

275 dtype=bool, 

276 default=False, 

277 ) 

278 multipleCyclesFinalCycleNumber = pexConfig.RangeField( 

279 doc=("Final cycle number in multiple cycle mode. The initial cycle " 

280 "is 0, with limited parameters fit. The next cycle is 1 with " 

281 "full parameter fit. The final cycle is a clean-up with no " 

282 "parameters fit. There will be a total of " 

283 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final " 

284 "cycle number cannot be less than 2."), 

285 dtype=int, 

286 default=5, 

287 min=2, 

288 max=MULTIPLE_CYCLES_MAX, 

289 inclusiveMax=True, 

290 ) 

291 bands = pexConfig.ListField( 

292 doc="Bands to run calibration", 

293 dtype=str, 

294 default=[], 

295 ) 

296 fitBands = pexConfig.ListField( 

297 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

298 "the atmosphere constrained from the 'fitBands' on the same night. " 

299 "Must be a subset of `config.bands`"), 

300 dtype=str, 

301 default=[], 

302 ) 

303 requiredBands = pexConfig.ListField( 

304 doc=("Bands that are required for a star to be considered a calibration star. " 

305 "Must be a subset of `config.bands`"), 

306 dtype=str, 

307 default=[], 

308 ) 

309 physicalFilterMap = pexConfig.DictField( 

310 doc="Mapping from 'physicalFilter' to band.", 

311 keytype=str, 

312 itemtype=str, 

313 default={}, 

314 ) 

315 doReferenceCalibration = pexConfig.Field( 

316 doc="Use reference catalog as additional constraint on calibration", 

317 dtype=bool, 

318 default=True, 

319 ) 

320 refStarSnMin = pexConfig.Field( 

321 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

322 dtype=float, 

323 default=50.0, 

324 ) 

325 refStarOutlierNSig = pexConfig.Field( 

326 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

327 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

328 dtype=float, 

329 default=4.0, 

330 ) 

331 applyRefStarColorCuts = pexConfig.Field( 

332 doc="Apply color cuts to reference stars?", 

333 dtype=bool, 

334 default=True, 

335 ) 

336 nCore = pexConfig.Field( 

337 doc="Number of cores to use", 

338 dtype=int, 

339 default=4, 

340 ) 

341 nStarPerRun = pexConfig.Field( 

342 doc="Number of stars to run in each chunk", 

343 dtype=int, 

344 default=200000, 

345 ) 

346 nExpPerRun = pexConfig.Field( 

347 doc="Number of exposures to run in each chunk", 

348 dtype=int, 

349 default=1000, 

350 ) 

351 reserveFraction = pexConfig.Field( 

352 doc="Fraction of stars to reserve for testing", 

353 dtype=float, 

354 default=0.1, 

355 ) 

356 freezeStdAtmosphere = pexConfig.Field( 

357 doc="Freeze atmosphere parameters to standard (for testing)", 

358 dtype=bool, 

359 default=False, 

360 ) 

361 precomputeSuperStarInitialCycle = pexConfig.Field( 

362 doc="Precompute superstar flat for initial cycle", 

363 dtype=bool, 

364 default=False, 

365 ) 

366 superStarSubCcdDict = pexConfig.DictField( 

367 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

368 "Must have one entry per band."), 

369 keytype=str, 

370 itemtype=bool, 

371 default={}, 

372 ) 

373 superStarSubCcdChebyshevOrder = pexConfig.Field( 

374 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

375 "Global default is first-order polynomials, and should be overridden " 

376 "on a camera-by-camera basis depending on the ISR."), 

377 dtype=int, 

378 default=1, 

379 ) 

380 superStarSubCcdTriangular = pexConfig.Field( 

381 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

382 "suppress high-order cross terms?"), 

383 dtype=bool, 

384 default=False, 

385 ) 

386 superStarSigmaClip = pexConfig.Field( 

387 doc="Number of sigma to clip outliers when selecting for superstar flats", 

388 dtype=float, 

389 default=5.0, 

390 ) 

391 focalPlaneSigmaClip = pexConfig.Field( 

392 doc="Number of sigma to clip outliers per focal-plane.", 

393 dtype=float, 

394 default=4.0, 

395 ) 

396 ccdGraySubCcdDict = pexConfig.DictField( 

397 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

398 "('ccd gray') on a sub-ccd scale."), 

399 keytype=str, 

400 itemtype=bool, 

401 default={}, 

402 ) 

403 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

404 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

405 dtype=int, 

406 default=1, 

407 ) 

408 ccdGraySubCcdTriangular = pexConfig.Field( 

409 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

410 "suppress high-order cross terms?"), 

411 dtype=bool, 

412 default=True, 

413 ) 

414 ccdGrayFocalPlaneDict = pexConfig.DictField( 

415 doc=("Per-band specification on whether to compute focal-plane residual " 

416 "('ccd gray') corrections."), 

417 keytype=str, 

418 itemtype=bool, 

419 default={}, 

420 ) 

421 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

422 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

423 "gray corrections. If there are fewer good CCDs then the gray " 

424 "correction is computed per-ccd."), 

425 dtype=int, 

426 default=1, 

427 ) 

428 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

429 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

430 dtype=int, 

431 default=3, 

432 ) 

433 cycleNumber = pexConfig.Field( 

434 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

435 "and stage of outlier rejection. See cookbook for details."), 

436 dtype=int, 

437 default=None, 

438 ) 

439 isFinalCycle = pexConfig.Field( 

440 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

441 "selection of stars and photometric exposures, and will output zeropoints " 

442 "and standard stars for use in fgcmOutputProducts"), 

443 dtype=bool, 

444 default=False, 

445 ) 

446 maxIterBeforeFinalCycle = pexConfig.Field( 

447 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

448 "will always be 0 in the final cycle for cleanup and final selection."), 

449 dtype=int, 

450 default=50, 

451 ) 

452 deltaMagBkgOffsetPercentile = pexConfig.Field( 

453 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

454 "offset from local background subtraction."), 

455 dtype=float, 

456 default=0.25, 

457 ) 

458 deltaMagBkgPerCcd = pexConfig.Field( 

459 doc=("Compute net offset from local background subtraction per-ccd? " 

460 "Otherwise, use computation per visit."), 

461 dtype=bool, 

462 default=False, 

463 ) 

464 utBoundary = pexConfig.Field( 

465 doc="Boundary (in UTC) from day-to-day", 

466 dtype=float, 

467 default=None, 

468 ) 

469 washMjds = pexConfig.ListField( 

470 doc="Mirror wash MJDs", 

471 dtype=float, 

472 default=(0.0,), 

473 ) 

474 epochMjds = pexConfig.ListField( 

475 doc="Epoch boundaries in MJD", 

476 dtype=float, 

477 default=(0.0,), 

478 ) 

479 minObsPerBand = pexConfig.Field( 

480 doc="Minimum good observations per band", 

481 dtype=int, 

482 default=2, 

483 ) 

484 # TODO: When DM-16511 is done, it will be possible to get the 

485 # telescope latitude directly from the camera. 

486 latitude = pexConfig.Field( 

487 doc="Observatory latitude", 

488 dtype=float, 

489 default=None, 

490 ) 

491 defaultCameraOrientation = pexConfig.Field( 

492 doc="Default camera orientation for QA plots.", 

493 dtype=float, 

494 default=None, 

495 ) 

496 brightObsGrayMax = pexConfig.Field( 

497 doc="Maximum gray extinction to be considered bright observation", 

498 dtype=float, 

499 default=0.15, 

500 ) 

501 minStarPerCcd = pexConfig.Field( 

502 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

503 "CCDs with fewer stars will have their calibration estimated from other " 

504 "CCDs in the same visit, with zeropoint error increased accordingly."), 

505 dtype=int, 

506 default=5, 

507 ) 

508 minCcdPerExp = pexConfig.Field( 

509 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

510 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

511 dtype=int, 

512 default=5, 

513 ) 

514 maxCcdGrayErr = pexConfig.Field( 

515 doc="Maximum error on CCD gray offset to be considered photometric", 

516 dtype=float, 

517 default=0.05, 

518 ) 

519 minStarPerExp = pexConfig.Field( 

520 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

521 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

522 dtype=int, 

523 default=600, 

524 ) 

525 minExpPerNight = pexConfig.Field( 

526 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

527 dtype=int, 

528 default=10, 

529 ) 

530 expGrayInitialCut = pexConfig.Field( 

531 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

532 "observations."), 

533 dtype=float, 

534 default=-0.25, 

535 ) 

536 expGrayPhotometricCutDict = pexConfig.DictField( 

537 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

538 "('gray term') for a visit to be considered photometric. Must have one " 

539 "entry per band. Broad-band filters should be -0.05."), 

540 keytype=str, 

541 itemtype=float, 

542 default={}, 

543 ) 

544 expGrayHighCutDict = pexConfig.DictField( 

545 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

546 "('gray term') for a visit to be considered photometric. Must have one " 

547 "entry per band. Broad-band filters should be 0.2."), 

548 keytype=str, 

549 itemtype=float, 

550 default={}, 

551 ) 

552 expGrayRecoverCut = pexConfig.Field( 

553 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

554 "Visits with more gray extinction will only get CCD zeropoints if there are " 

555 "sufficient star observations (minStarPerCcd) on that CCD."), 

556 dtype=float, 

557 default=-1.0, 

558 ) 

559 expVarGrayPhotometricCutDict = pexConfig.DictField( 

560 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

561 "photometric. Must have one entry per band. Broad-band filters should be " 

562 "0.0005."), 

563 keytype=str, 

564 itemtype=float, 

565 default={}, 

566 ) 

567 expGrayErrRecoverCut = pexConfig.Field( 

568 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

569 "Visits with more gray variance will only get CCD zeropoints if there are " 

570 "sufficient star observations (minStarPerCcd) on that CCD."), 

571 dtype=float, 

572 default=0.05, 

573 ) 

574 aperCorrFitNBins = pexConfig.Field( 

575 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

576 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

577 "used if available."), 

578 dtype=int, 

579 default=10, 

580 ) 

581 aperCorrInputSlopeDict = pexConfig.DictField( 

582 doc=("Per-band specification of aperture correction input slope parameters. These " 

583 "are used on the first fit iteration, and aperture correction parameters will " 

584 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

585 "to set this when there is insufficient data to fit the parameters (e.g. " 

586 "tract mode)."), 

587 keytype=str, 

588 itemtype=float, 

589 default={}, 

590 ) 

591 sedboundaryterms = pexConfig.ConfigField( 

592 doc="Mapping from bands to SED boundary term names used is sedterms.", 

593 dtype=SedboundarytermDict, 

594 ) 

595 sedterms = pexConfig.ConfigField( 

596 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

597 dtype=SedtermDict, 

598 ) 

599 sigFgcmMaxErr = pexConfig.Field( 

600 doc="Maximum mag error for fitting sigma_FGCM", 

601 dtype=float, 

602 default=0.01, 

603 ) 

604 sigFgcmMaxEGrayDict = pexConfig.DictField( 

605 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

606 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

607 "should be 0.05."), 

608 keytype=str, 

609 itemtype=float, 

610 default={}, 

611 ) 

612 ccdGrayMaxStarErr = pexConfig.Field( 

613 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

614 "computation"), 

615 dtype=float, 

616 default=0.10, 

617 ) 

618 approxThroughputDict = pexConfig.DictField( 

619 doc=("Per-band specification of the approximate overall throughput at the start of " 

620 "calibration observations. Must have one entry per band. Typically should " 

621 "be 1.0."), 

622 keytype=str, 

623 itemtype=float, 

624 default={}, 

625 ) 

626 sigmaCalRange = pexConfig.ListField( 

627 doc="Allowed range for systematic error floor estimation", 

628 dtype=float, 

629 default=(0.001, 0.003), 

630 ) 

631 sigmaCalFitPercentile = pexConfig.ListField( 

632 doc="Magnitude percentile range to fit systematic error floor", 

633 dtype=float, 

634 default=(0.05, 0.15), 

635 ) 

636 sigmaCalPlotPercentile = pexConfig.ListField( 

637 doc="Magnitude percentile range to plot systematic error floor", 

638 dtype=float, 

639 default=(0.05, 0.95), 

640 ) 

641 sigma0Phot = pexConfig.Field( 

642 doc="Systematic error floor for all zeropoints", 

643 dtype=float, 

644 default=0.003, 

645 ) 

646 mapLongitudeRef = pexConfig.Field( 

647 doc="Reference longitude for plotting maps", 

648 dtype=float, 

649 default=0.0, 

650 ) 

651 mapNSide = pexConfig.Field( 

652 doc="Healpix nside for plotting maps", 

653 dtype=int, 

654 default=256, 

655 ) 

656 outfileBase = pexConfig.Field( 

657 doc="Filename start for plot output files", 

658 dtype=str, 

659 default=None, 

660 ) 

661 starColorCuts = pexConfig.ListField( 

662 doc="Encoded star-color cuts (to be cleaned up)", 

663 dtype=str, 

664 default=("NO_DATA",), 

665 ) 

666 colorSplitBands = pexConfig.ListField( 

667 doc="Band names to use to split stars by color. Must have 2 entries.", 

668 dtype=str, 

669 length=2, 

670 default=('g', 'i'), 

671 ) 

672 modelMagErrors = pexConfig.Field( 

673 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

674 dtype=bool, 

675 default=True, 

676 ) 

677 useQuadraticPwv = pexConfig.Field( 

678 doc="Model PWV with a quadratic term for variation through the night?", 

679 dtype=bool, 

680 default=False, 

681 ) 

682 instrumentParsPerBand = pexConfig.Field( 

683 doc=("Model instrumental parameters per band? " 

684 "Otherwise, instrumental parameters (QE changes with time) are " 

685 "shared among all bands."), 

686 dtype=bool, 

687 default=False, 

688 ) 

689 instrumentSlopeMinDeltaT = pexConfig.Field( 

690 doc=("Minimum time change (in days) between observations to use in constraining " 

691 "instrument slope."), 

692 dtype=float, 

693 default=20.0, 

694 ) 

695 fitMirrorChromaticity = pexConfig.Field( 

696 doc="Fit (intraband) mirror chromatic term?", 

697 dtype=bool, 

698 default=False, 

699 ) 

700 coatingMjds = pexConfig.ListField( 

701 doc="Mirror coating dates in MJD", 

702 dtype=float, 

703 default=(0.0,), 

704 ) 

705 outputStandardsBeforeFinalCycle = pexConfig.Field( 

706 doc="Output standard stars prior to final cycle? Used in debugging.", 

707 dtype=bool, 

708 default=False, 

709 ) 

710 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

711 doc="Output standard stars prior to final cycle? Used in debugging.", 

712 dtype=bool, 

713 default=False, 

714 ) 

715 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

716 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

717 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

718 keytype=str, 

719 itemtype=bool, 

720 default={}, 

721 ) 

722 autoPhotometricCutNSig = pexConfig.Field( 

723 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

724 "Cut is based on exposure gray width (per band), unless " 

725 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

726 "repeatability is used (also per band)."), 

727 dtype=float, 

728 default=3.0, 

729 ) 

730 autoHighCutNSig = pexConfig.Field( 

731 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

732 "Cut is based on exposure gray width (per band), unless " 

733 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

734 "repeatability is used (also per band)."), 

735 dtype=float, 

736 default=4.0, 

737 ) 

738 quietMode = pexConfig.Field( 

739 doc="Be less verbose with logging.", 

740 dtype=bool, 

741 default=False, 

742 ) 

743 doPlots = pexConfig.Field( 

744 doc="Make fgcm QA plots.", 

745 dtype=bool, 

746 default=True, 

747 ) 

748 randomSeed = pexConfig.Field( 

749 doc="Random seed for fgcm for consistency in tests.", 

750 dtype=int, 

751 default=None, 

752 optional=True, 

753 ) 

754 deltaAperFitMinNgoodObs = pexConfig.Field( 

755 doc="Minimum number of good observations to use mean delta-aper values in fits.", 

756 dtype=int, 

757 default=2, 

758 ) 

759 deltaAperFitPerCcdNx = pexConfig.Field( 

760 doc=("Number of x bins per ccd when computing delta-aper background offsets. " 

761 "Only used when ``doComputeDeltaAperPerCcd`` is True."), 

762 dtype=int, 

763 default=10, 

764 ) 

765 deltaAperFitPerCcdNy = pexConfig.Field( 

766 doc=("Number of y bins per ccd when computing delta-aper background offsets. " 

767 "Only used when ``doComputeDeltaAperPerCcd`` is True."), 

768 dtype=int, 

769 default=10, 

770 ) 

771 deltaAperFitSpatialNside = pexConfig.Field( 

772 doc="Healpix nside to compute spatial delta-aper background offset maps.", 

773 dtype=int, 

774 default=64, 

775 ) 

776 deltaAperInnerRadiusArcsec = pexConfig.Field( 

777 doc=("Inner radius used to compute deltaMagAper (arcseconds). " 

778 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if " 

779 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, " 

780 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."), 

781 dtype=float, 

782 default=0.0, 

783 ) 

784 deltaAperOuterRadiusArcsec = pexConfig.Field( 

785 doc=("Outer radius used to compute deltaMagAper (arcseconds). " 

786 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if " 

787 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, " 

788 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."), 

789 dtype=float, 

790 default=0.0, 

791 ) 

792 doComputeDeltaAperPerVisit = pexConfig.Field( 

793 doc=("Do the computation of delta-aper background offsets per visit? " 

794 "Note: this option can be very slow when there are many visits."), 

795 dtype=bool, 

796 default=False, 

797 ) 

798 doComputeDeltaAperPerStar = pexConfig.Field( 

799 doc="Do the computation of delta-aper mean values per star?", 

800 dtype=bool, 

801 default=True, 

802 ) 

803 doComputeDeltaAperMap = pexConfig.Field( 

804 doc=("Do the computation of delta-aper spatial maps? " 

805 "This is only used if ``doComputeDeltaAperPerStar`` is True,"), 

806 dtype=bool, 

807 default=False, 

808 ) 

809 doComputeDeltaAperPerCcd = pexConfig.Field( 

810 doc="Do the computation of per-ccd delta-aper background offsets?", 

811 dtype=bool, 

812 default=False, 

813 ) 

814 

815 def validate(self): 

816 super().validate() 

817 

818 if self.connections.previousCycleNumber != str(self.cycleNumber - 1): 

819 msg = "cycleNumber in template must be connections.previousCycleNumber + 1" 

820 raise RuntimeError(msg) 

821 if self.connections.cycleNumber != str(self.cycleNumber): 

822 msg = "cycleNumber in template must be equal to connections.cycleNumber" 

823 raise RuntimeError(msg) 

824 

825 for band in self.fitBands: 

826 if band not in self.bands: 

827 msg = 'fitBand %s not in bands' % (band) 

828 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

829 for band in self.requiredBands: 

830 if band not in self.bands: 

831 msg = 'requiredBand %s not in bands' % (band) 

832 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

833 for band in self.colorSplitBands: 

834 if band not in self.bands: 

835 msg = 'colorSplitBand %s not in bands' % (band) 

836 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

837 for band in self.bands: 

838 if band not in self.superStarSubCcdDict: 

839 msg = 'band %s not in superStarSubCcdDict' % (band) 

840 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

841 self, msg) 

842 if band not in self.ccdGraySubCcdDict: 

843 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

844 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

845 self, msg) 

846 if band not in self.expGrayPhotometricCutDict: 

847 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

848 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

849 self, msg) 

850 if band not in self.expGrayHighCutDict: 

851 msg = 'band %s not in expGrayHighCutDict' % (band) 

852 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

853 self, msg) 

854 if band not in self.expVarGrayPhotometricCutDict: 

855 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

856 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

857 self, msg) 

858 if band not in self.sigFgcmMaxEGrayDict: 

859 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

860 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

861 self, msg) 

862 if band not in self.approxThroughputDict: 

863 msg = 'band %s not in approxThroughputDict' % (band) 

864 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

865 self, msg) 

866 if band not in self.useRepeatabilityForExpGrayCutsDict: 

867 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

868 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

869 self, msg) 

870 

871 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \ 

872 or self.doComputeDeltaAperPerCcd: 

873 if self.deltaAperInnerRadiusArcsec <= 0.0: 

874 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.' 

875 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec, 

876 self, msg) 

877 if self.deltaAperOuterRadiusArcsec <= 0.0: 

878 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.' 

879 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec, 

880 self, msg) 

881 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec: 

882 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if ' 

883 'deltaAper computations are turned on.') 

884 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec, 

885 self, msg) 

886 

887 

888class FgcmFitCycleTask(pipeBase.PipelineTask): 

889 """ 

890 Run Single fit cycle for FGCM global calibration 

891 """ 

892 

893 ConfigClass = FgcmFitCycleConfig 

894 _DefaultName = "fgcmFitCycle" 

895 

896 def __init__(self, initInputs=None, **kwargs): 

897 super().__init__(**kwargs) 

898 

899 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

900 camera = butlerQC.get(inputRefs.camera) 

901 

902 handleDict = {} 

903 

904 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

905 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

906 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations) 

907 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds) 

908 handleDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices) 

909 if self.config.doReferenceCalibration: 

910 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars) 

911 if self.config.cycleNumber > 0: 

912 handleDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput) 

913 handleDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput) 

914 

915 fgcmDatasetDict = None 

916 if self.config.doMultipleCycles: 

917 # Run multiple cycles at once. 

918 config = copy.copy(self.config) 

919 config.update(cycleNumber=0) 

920 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1): 

921 if cycle == self.config.multipleCyclesFinalCycleNumber: 

922 config.update(isFinalCycle=True) 

923 

924 if cycle > 0: 

925 handleDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars'] 

926 handleDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters'] 

927 

928 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config) 

929 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], 

930 getattr(outputRefs, f'fgcmFitParameters{cycle}')) 

931 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], 

932 getattr(outputRefs, f'fgcmFlaggedStars{cycle}')) 

933 if self.outputZeropoints: 

934 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], 

935 getattr(outputRefs, f'fgcmZeropoints{cycle}')) 

936 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 

937 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}')) 

938 if self.outputStandards: 

939 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], 

940 getattr(outputRefs, f'fgcmStandardStars{cycle}')) 

941 else: 

942 # Run a single cycle 

943 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict) 

944 

945 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters) 

946 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars) 

947 if self.outputZeropoints: 

948 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints) 

949 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters) 

950 if self.outputStandards: 

951 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars) 

952 

953 def _fgcmFitCycle(self, camera, handleDict, config=None): 

954 """ 

955 Run the fit cycle 

956 

957 Parameters 

958 ---------- 

959 camera : `lsst.afw.cameraGeom.Camera` 

960 handleDict : `dict` 

961 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

962 handle dictionary with keys: 

963 

964 ``"fgcmLookUpTable"`` 

965 handle for the FGCM look-up table. 

966 ``"fgcmVisitCatalog"`` 

967 handle for visit summary catalog. 

968 ``"fgcmStarObservations"`` 

969 handle for star observation catalog. 

970 ``"fgcmStarIds"`` 

971 handle for star id catalog. 

972 ``"fgcmStarIndices"`` 

973 handle for star index catalog. 

974 ``"fgcmReferenceStars"`` 

975 handle for matched reference star catalog. 

976 ``"fgcmFlaggedStars"`` 

977 handle for flagged star catalog. 

978 ``"fgcmFitParameters"`` 

979 handle for fit parameter catalog. 

980 config : `lsst.pex.config.Config`, optional 

981 Configuration to use to override self.config. 

982 

983 Returns 

984 ------- 

985 fgcmDatasetDict : `dict` 

986 Dictionary of datasets to persist. 

987 """ 

988 if config is not None: 

989 _config = config 

990 else: 

991 _config = self.config 

992 

993 # Set defaults on whether to output standards and zeropoints 

994 self.maxIter = _config.maxIterBeforeFinalCycle 

995 self.outputStandards = _config.outputStandardsBeforeFinalCycle 

996 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle 

997 self.resetFitParameters = True 

998 

999 if _config.isFinalCycle: 

1000 # This is the final fit cycle, so we do not want to reset fit 

1001 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

1002 # and we always want to output standards and zeropoints 

1003 self.maxIter = 0 

1004 self.outputStandards = True 

1005 self.outputZeropoints = True 

1006 self.resetFitParameters = False 

1007 

1008 lutCat = handleDict['fgcmLookUpTable'].get() 

1009 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

1010 dict(_config.physicalFilterMap)) 

1011 del lutCat 

1012 

1013 configDict = makeConfigDict(_config, self.log, camera, 

1014 self.maxIter, self.resetFitParameters, 

1015 self.outputZeropoints, 

1016 lutIndexVals[0]['FILTERNAMES']) 

1017 

1018 # next we need the exposure/visit information 

1019 visitCat = handleDict['fgcmVisitCatalog'].get() 

1020 fgcmExpInfo = translateVisitCatalog(visitCat) 

1021 del visitCat 

1022 

1023 focalPlaneProjector = FocalPlaneProjector(camera, 

1024 self.config.defaultCameraOrientation) 

1025 

1026 noFitsDict = {'lutIndex': lutIndexVals, 

1027 'lutStd': lutStd, 

1028 'expInfo': fgcmExpInfo, 

1029 'focalPlaneProjector': focalPlaneProjector} 

1030 

1031 # set up the fitter object 

1032 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

1033 noFitsDict=noFitsDict, noOutput=True) 

1034 

1035 # create the parameter object 

1036 if (fgcmFitCycle.initialCycle): 

1037 # cycle = 0, initial cycle 

1038 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

1039 fgcmLut, 

1040 fgcmExpInfo) 

1041 else: 

1042 if isinstance(handleDict['fgcmFitParameters'], afwTable.BaseCatalog): 

1043 parCat = handleDict['fgcmFitParameters'] 

1044 else: 

1045 parCat = handleDict['fgcmFitParameters'].get() 

1046 inParInfo, inParams, inSuperStar = self._loadParameters(parCat) 

1047 del parCat 

1048 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

1049 fgcmExpInfo, 

1050 inParInfo, 

1051 inParams, 

1052 inSuperStar) 

1053 

1054 # set up the stars... 

1055 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

1056 

1057 starObs = handleDict['fgcmStarObservations'].get() 

1058 starIds = handleDict['fgcmStarIds'].get() 

1059 starIndices = handleDict['fgcmStarIndices'].get() 

1060 

1061 # grab the flagged stars if available 

1062 if 'fgcmFlaggedStars' in handleDict: 

1063 if isinstance(handleDict['fgcmFlaggedStars'], afwTable.BaseCatalog): 

1064 flaggedStars = handleDict['fgcmFlaggedStars'] 

1065 else: 

1066 flaggedStars = handleDict['fgcmFlaggedStars'].get() 

1067 flagId = flaggedStars['objId'][:] 

1068 flagFlag = flaggedStars['objFlag'][:] 

1069 else: 

1070 flaggedStars = None 

1071 flagId = None 

1072 flagFlag = None 

1073 

1074 if _config.doReferenceCalibration: 

1075 refStars = handleDict['fgcmReferenceStars'].get() 

1076 

1077 refMag, refMagErr = extractReferenceMags(refStars, 

1078 _config.bands, 

1079 _config.physicalFilterMap) 

1080 refId = refStars['fgcm_id'][:] 

1081 else: 

1082 refStars = None 

1083 refId = None 

1084 refMag = None 

1085 refMagErr = None 

1086 

1087 # match star observations to visits 

1088 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

1089 # actually be transferred into fgcm using the indexing below. 

1090 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

1091 

1092 # The fgcmStars.loadStars method will copy all the star information into 

1093 # special shared memory objects that will not blow up the memory usage when 

1094 # used with python multiprocessing. Once all the numbers are copied, 

1095 # it is necessary to release all references to the objects that previously 

1096 # stored the data to ensure that the garbage collector can clear the memory, 

1097 # and ensure that this memory is not copied when multiprocessing kicks in. 

1098 

1099 # We determine the conversion from the native units (typically radians) to 

1100 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

1101 # numpy arrays rather than Angles, which would we approximately 600x slower. 

1102 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

1103 

1104 fgcmStars.loadStars(fgcmPars, 

1105 starObs['visit'][starIndices['obsIndex']], 

1106 starObs['ccd'][starIndices['obsIndex']], 

1107 starObs['ra'][starIndices['obsIndex']] * conv, 

1108 starObs['dec'][starIndices['obsIndex']] * conv, 

1109 starObs['instMag'][starIndices['obsIndex']], 

1110 starObs['instMagErr'][starIndices['obsIndex']], 

1111 fgcmExpInfo['FILTERNAME'][visitIndex], 

1112 starIds['fgcm_id'][:], 

1113 starIds['ra'][:], 

1114 starIds['dec'][:], 

1115 starIds['obsArrIndex'][:], 

1116 starIds['nObs'][:], 

1117 obsX=starObs['x'][starIndices['obsIndex']], 

1118 obsY=starObs['y'][starIndices['obsIndex']], 

1119 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']], 

1120 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']], 

1121 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

1122 refID=refId, 

1123 refMag=refMag, 

1124 refMagErr=refMagErr, 

1125 flagID=flagId, 

1126 flagFlag=flagFlag, 

1127 computeNobs=True) 

1128 

1129 # Release all references to temporary objects holding star data (see above) 

1130 del starObs 

1131 del starIds 

1132 del starIndices 

1133 del flagId 

1134 del flagFlag 

1135 del flaggedStars 

1136 del refStars 

1137 del refId 

1138 del refMag 

1139 del refMagErr 

1140 

1141 # and set the bits in the cycle object 

1142 fgcmFitCycle.setLUT(fgcmLut) 

1143 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

1144 fgcmFitCycle.setPars(fgcmPars) 

1145 

1146 # finish the setup 

1147 fgcmFitCycle.finishSetup() 

1148 

1149 # and run 

1150 fgcmFitCycle.run() 

1151 

1152 ################## 

1153 # Persistance 

1154 ################## 

1155 

1156 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle) 

1157 

1158 # Output the config for the next cycle 

1159 # We need to make a copy since the input one has been frozen 

1160 

1161 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

1162 i, b in enumerate(_config.bands)} 

1163 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

1164 i, band in enumerate(_config.bands)} 

1165 

1166 outConfig = copy.copy(_config) 

1167 outConfig.update(cycleNumber=(_config.cycleNumber + 1), 

1168 precomputeSuperStarInitialCycle=False, 

1169 freezeStdAtmosphere=False, 

1170 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

1171 expGrayHighCutDict=updatedHighCutDict) 

1172 

1173 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber), 

1174 cycleNumber=str(_config.cycleNumber + 1)) 

1175 

1176 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

1177 outConfig.cycleNumber) 

1178 outConfig.save(configFileName) 

1179 

1180 if _config.isFinalCycle == 1: 

1181 # We are done, ready to output products 

1182 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

1183 else: 

1184 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

1185 self.log.info("Be sure to look at:") 

1186 self.log.info(" config.expGrayPhotometricCut") 

1187 self.log.info(" config.expGrayHighCut") 

1188 self.log.info("If you are satisfied with the fit, please set:") 

1189 self.log.info(" config.isFinalCycle = True") 

1190 

1191 fgcmFitCycle.freeSharedMemory() 

1192 

1193 return fgcmDatasetDict, outConfig 

1194 

1195 def _loadParameters(self, parCat): 

1196 """ 

1197 Load FGCM parameters from a previous fit cycle 

1198 

1199 Parameters 

1200 ---------- 

1201 parCat : `lsst.afw.table.BaseCatalog` 

1202 Parameter catalog in afw table form. 

1203 

1204 Returns 

1205 ------- 

1206 inParInfo: `numpy.ndarray` 

1207 Numpy array parameter information formatted for input to fgcm 

1208 inParameters: `numpy.ndarray` 

1209 Numpy array parameter values formatted for input to fgcm 

1210 inSuperStar: `numpy.array` 

1211 Superstar flat formatted for input to fgcm 

1212 """ 

1213 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1214 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1215 

1216 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1217 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1218 (parLutFilterNames.size, )), 

1219 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1220 ('LNTAUUNIT', 'f8'), 

1221 ('LNTAUSLOPEUNIT', 'f8'), 

1222 ('ALPHAUNIT', 'f8'), 

1223 ('LNPWVUNIT', 'f8'), 

1224 ('LNPWVSLOPEUNIT', 'f8'), 

1225 ('LNPWVQUADRATICUNIT', 'f8'), 

1226 ('LNPWVGLOBALUNIT', 'f8'), 

1227 ('O3UNIT', 'f8'), 

1228 ('QESYSUNIT', 'f8'), 

1229 ('FILTEROFFSETUNIT', 'f8'), 

1230 ('HASEXTERNALPWV', 'i2'), 

1231 ('HASEXTERNALTAU', 'i2')]) 

1232 inParInfo['NCCD'] = parCat['nCcd'] 

1233 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1234 inParInfo['FITBANDS'][:] = parFitBands 

1235 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1236 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1237 

1238 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1239 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1240 ('PARLNTAUINTERCEPT', 'f8', 

1241 (parCat['parLnTauIntercept'].size, )), 

1242 ('PARLNTAUSLOPE', 'f8', 

1243 (parCat['parLnTauSlope'].size, )), 

1244 ('PARLNPWVINTERCEPT', 'f8', 

1245 (parCat['parLnPwvIntercept'].size, )), 

1246 ('PARLNPWVSLOPE', 'f8', 

1247 (parCat['parLnPwvSlope'].size, )), 

1248 ('PARLNPWVQUADRATIC', 'f8', 

1249 (parCat['parLnPwvQuadratic'].size, )), 

1250 ('PARQESYSINTERCEPT', 'f8', 

1251 (parCat['parQeSysIntercept'].size, )), 

1252 ('COMPQESYSSLOPE', 'f8', 

1253 (parCat['compQeSysSlope'].size, )), 

1254 ('PARFILTEROFFSET', 'f8', 

1255 (parCat['parFilterOffset'].size, )), 

1256 ('PARFILTEROFFSETFITFLAG', 'i2', 

1257 (parCat['parFilterOffsetFitFlag'].size, )), 

1258 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1259 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1260 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1261 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1262 ('COMPABSTHROUGHPUT', 'f8', 

1263 (parCat['compAbsThroughput'].size, )), 

1264 ('COMPREFOFFSET', 'f8', 

1265 (parCat['compRefOffset'].size, )), 

1266 ('COMPREFSIGMA', 'f8', 

1267 (parCat['compRefSigma'].size, )), 

1268 ('COMPMIRRORCHROMATICITY', 'f8', 

1269 (parCat['compMirrorChromaticity'].size, )), 

1270 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1271 (parCat['mirrorChromaticityPivot'].size, )), 

1272 ('COMPMEDIANSEDSLOPE', 'f8', 

1273 (parCat['compMedianSedSlope'].size, )), 

1274 ('COMPAPERCORRPIVOT', 'f8', 

1275 (parCat['compAperCorrPivot'].size, )), 

1276 ('COMPAPERCORRSLOPE', 'f8', 

1277 (parCat['compAperCorrSlope'].size, )), 

1278 ('COMPAPERCORRSLOPEERR', 'f8', 

1279 (parCat['compAperCorrSlopeErr'].size, )), 

1280 ('COMPAPERCORRRANGE', 'f8', 

1281 (parCat['compAperCorrRange'].size, )), 

1282 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1283 (parCat['compModelErrExptimePivot'].size, )), 

1284 ('COMPMODELERRFWHMPIVOT', 'f8', 

1285 (parCat['compModelErrFwhmPivot'].size, )), 

1286 ('COMPMODELERRSKYPIVOT', 'f8', 

1287 (parCat['compModelErrSkyPivot'].size, )), 

1288 ('COMPMODELERRPARS', 'f8', 

1289 (parCat['compModelErrPars'].size, )), 

1290 ('COMPEXPGRAY', 'f8', 

1291 (parCat['compExpGray'].size, )), 

1292 ('COMPVARGRAY', 'f8', 

1293 (parCat['compVarGray'].size, )), 

1294 ('COMPEXPDELTAMAGBKG', 'f8', 

1295 (parCat['compExpDeltaMagBkg'].size, )), 

1296 ('COMPNGOODSTARPEREXP', 'i4', 

1297 (parCat['compNGoodStarPerExp'].size, )), 

1298 ('COMPSIGFGCM', 'f8', 

1299 (parCat['compSigFgcm'].size, )), 

1300 ('COMPSIGMACAL', 'f8', 

1301 (parCat['compSigmaCal'].size, )), 

1302 ('COMPRETRIEVEDLNPWV', 'f8', 

1303 (parCat['compRetrievedLnPwv'].size, )), 

1304 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1305 (parCat['compRetrievedLnPwvRaw'].size, )), 

1306 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1307 (parCat['compRetrievedLnPwvFlag'].size, )), 

1308 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1309 (parCat['compRetrievedTauNight'].size, )), 

1310 ('COMPEPSILON', 'f8', 

1311 (parCat['compEpsilon'].size, )), 

1312 ('COMPMEDDELTAAPER', 'f8', 

1313 (parCat['compMedDeltaAper'].size, )), 

1314 ('COMPGLOBALEPSILON', 'f4', 

1315 (parCat['compGlobalEpsilon'].size, )), 

1316 ('COMPEPSILONMAP', 'f4', 

1317 (parCat['compEpsilonMap'].size, )), 

1318 ('COMPEPSILONNSTARMAP', 'i4', 

1319 (parCat['compEpsilonNStarMap'].size, )), 

1320 ('COMPEPSILONCCDMAP', 'f4', 

1321 (parCat['compEpsilonCcdMap'].size, )), 

1322 ('COMPEPSILONCCDNSTARMAP', 'i4', 

1323 (parCat['compEpsilonCcdNStarMap'].size, ))]) 

1324 

1325 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1326 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1327 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1328 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1329 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1330 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1331 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1332 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1333 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1334 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1335 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1336 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1337 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1338 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1339 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1340 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1341 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1342 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1343 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1344 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :] 

1345 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1346 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1347 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1348 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1349 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1350 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1351 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1352 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1353 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1354 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1355 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :] 

1356 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1357 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1358 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1359 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1360 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1361 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1362 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1363 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :] 

1364 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :] 

1365 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :] 

1366 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :] 

1367 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :] 

1368 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :] 

1369 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :] 

1370 

1371 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1372 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1373 

1374 return (inParInfo, inParams, inSuperStar) 

1375 

1376 def _makeFgcmOutputDatasets(self, fgcmFitCycle): 

1377 """ 

1378 Persist FGCM datasets through the butler. 

1379 

1380 Parameters 

1381 ---------- 

1382 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1383 Fgcm Fit cycle object 

1384 """ 

1385 fgcmDatasetDict = {} 

1386 

1387 # Save the parameters 

1388 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1389 

1390 parSchema = afwTable.Schema() 

1391 

1392 comma = ',' 

1393 lutFilterNameString = comma.join([n.decode('utf-8') 

1394 for n in parInfo['LUTFILTERNAMES'][0]]) 

1395 fitBandString = comma.join([n.decode('utf-8') 

1396 for n in parInfo['FITBANDS'][0]]) 

1397 

1398 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1399 lutFilterNameString, fitBandString) 

1400 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1401 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1402 lutFilterNameString, fitBandString) 

1403 

1404 fgcmDatasetDict['fgcmFitParameters'] = parCat 

1405 

1406 # Save the indices of the flagged stars 

1407 # (stars that have been (a) reserved from the fit for testing and 

1408 # (b) bad stars that have failed quality checks.) 

1409 flagStarSchema = self._makeFlagStarSchema() 

1410 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1411 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1412 

1413 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat 

1414 

1415 # Save the zeropoint information and atmospheres only if desired 

1416 if self.outputZeropoints: 

1417 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1418 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1419 

1420 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1421 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1422 

1423 fgcmDatasetDict['fgcmZeropoints'] = zptCat 

1424 

1425 # Save atmosphere values 

1426 # These are generated by the same code that generates zeropoints 

1427 atmSchema = makeAtmSchema() 

1428 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1429 

1430 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat 

1431 

1432 # Save the standard stars (if configured) 

1433 if self.outputStandards: 

1434 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1435 stdSchema = makeStdSchema(len(goodBands)) 

1436 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1437 

1438 fgcmDatasetDict['fgcmStandardStars'] = stdCat 

1439 

1440 return fgcmDatasetDict 

1441 

1442 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1443 lutFilterNameString, fitBandString): 

1444 """ 

1445 Make the parameter persistence schema 

1446 

1447 Parameters 

1448 ---------- 

1449 parInfo: `numpy.ndarray` 

1450 Parameter information returned by fgcm 

1451 pars: `numpy.ndarray` 

1452 Parameter values returned by fgcm 

1453 parSuperStarFlat: `numpy.array` 

1454 Superstar flat values returned by fgcm 

1455 lutFilterNameString: `str` 

1456 Combined string of all the lutFilterNames 

1457 fitBandString: `str` 

1458 Combined string of all the fitBands 

1459 

1460 Returns 

1461 ------- 

1462 parSchema: `afwTable.schema` 

1463 """ 

1464 

1465 parSchema = afwTable.Schema() 

1466 

1467 # parameter info section 

1468 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1469 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1470 size=len(lutFilterNameString)) 

1471 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1472 size=len(fitBandString)) 

1473 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1474 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1475 doc='Step units for ln(AOD) slope') 

1476 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1477 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1478 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1479 doc='Step units for ln(pwv) slope') 

1480 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1481 doc='Step units for ln(pwv) quadratic term') 

1482 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1483 doc='Step units for global ln(pwv) parameters') 

1484 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1485 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1486 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1487 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1488 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1489 

1490 # parameter section 

1491 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1492 size=pars['PARALPHA'].size) 

1493 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1494 size=pars['PARO3'].size) 

1495 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1496 doc='ln(Tau) intercept parameter vector', 

1497 size=pars['PARLNTAUINTERCEPT'].size) 

1498 parSchema.addField('parLnTauSlope', type='ArrayD', 

1499 doc='ln(Tau) slope parameter vector', 

1500 size=pars['PARLNTAUSLOPE'].size) 

1501 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1502 size=pars['PARLNPWVINTERCEPT'].size) 

1503 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1504 size=pars['PARLNPWVSLOPE'].size) 

1505 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1506 size=pars['PARLNPWVQUADRATIC'].size) 

1507 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1508 size=pars['PARQESYSINTERCEPT'].size) 

1509 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1510 size=pars[0]['COMPQESYSSLOPE'].size) 

1511 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1512 size=pars['PARFILTEROFFSET'].size) 

1513 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1514 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1515 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1516 doc='Global scale for retrieved ln(pwv)') 

1517 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1518 doc='Global offset for retrieved ln(pwv)') 

1519 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1520 doc='Nightly offset for retrieved ln(pwv)', 

1521 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1522 parSchema.addField('compAbsThroughput', type='ArrayD', 

1523 doc='Absolute throughput (relative to transmission curves)', 

1524 size=pars['COMPABSTHROUGHPUT'].size) 

1525 parSchema.addField('compRefOffset', type='ArrayD', 

1526 doc='Offset between reference stars and calibrated stars', 

1527 size=pars['COMPREFOFFSET'].size) 

1528 parSchema.addField('compRefSigma', type='ArrayD', 

1529 doc='Width of reference star/calibrated star distribution', 

1530 size=pars['COMPREFSIGMA'].size) 

1531 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1532 doc='Computed mirror chromaticity terms', 

1533 size=pars['COMPMIRRORCHROMATICITY'].size) 

1534 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1535 doc='Mirror chromaticity pivot mjd', 

1536 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1537 parSchema.addField('compMedianSedSlope', type='ArrayD', 

1538 doc='Computed median SED slope (per band)', 

1539 size=pars['COMPMEDIANSEDSLOPE'].size) 

1540 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1541 size=pars['COMPAPERCORRPIVOT'].size) 

1542 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1543 size=pars['COMPAPERCORRSLOPE'].size) 

1544 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1545 size=pars['COMPAPERCORRSLOPEERR'].size) 

1546 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1547 size=pars['COMPAPERCORRRANGE'].size) 

1548 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1549 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1550 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1551 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1552 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1553 size=pars['COMPMODELERRSKYPIVOT'].size) 

1554 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1555 size=pars['COMPMODELERRPARS'].size) 

1556 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1557 size=pars['COMPEXPGRAY'].size) 

1558 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1559 size=pars['COMPVARGRAY'].size) 

1560 parSchema.addField('compExpDeltaMagBkg', type='ArrayD', 

1561 doc='Computed exposure offset due to background', 

1562 size=pars['COMPEXPDELTAMAGBKG'].size) 

1563 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1564 doc='Computed number of good stars per exposure', 

1565 size=pars['COMPNGOODSTARPEREXP'].size) 

1566 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1567 size=pars['COMPSIGFGCM'].size) 

1568 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1569 size=pars['COMPSIGMACAL'].size) 

1570 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1571 size=pars['COMPRETRIEVEDLNPWV'].size) 

1572 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1573 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1574 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1575 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1576 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1577 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1578 parSchema.addField('compEpsilon', type='ArrayD', 

1579 doc='Computed epsilon background offset per visit (nJy/arcsec2)', 

1580 size=pars['COMPEPSILON'].size) 

1581 parSchema.addField('compMedDeltaAper', type='ArrayD', 

1582 doc='Median delta mag aper per visit', 

1583 size=pars['COMPMEDDELTAAPER'].size) 

1584 parSchema.addField('compGlobalEpsilon', type='ArrayD', 

1585 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)', 

1586 size=pars['COMPGLOBALEPSILON'].size) 

1587 parSchema.addField('compEpsilonMap', type='ArrayD', 

1588 doc='Computed epsilon maps (nJy/arcsec2)', 

1589 size=pars['COMPEPSILONMAP'].size) 

1590 parSchema.addField('compEpsilonNStarMap', type='ArrayI', 

1591 doc='Number of stars per pixel in computed epsilon maps', 

1592 size=pars['COMPEPSILONNSTARMAP'].size) 

1593 parSchema.addField('compEpsilonCcdMap', type='ArrayD', 

1594 doc='Computed epsilon ccd maps (nJy/arcsec2)', 

1595 size=pars['COMPEPSILONCCDMAP'].size) 

1596 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI', 

1597 doc='Number of stars per ccd bin in epsilon ccd maps', 

1598 size=pars['COMPEPSILONCCDNSTARMAP'].size) 

1599 # superstarflat section 

1600 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1601 size=4) 

1602 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1603 size=parSuperStarFlat.size) 

1604 

1605 return parSchema 

1606 

1607 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1608 lutFilterNameString, fitBandString): 

1609 """ 

1610 Make the FGCM parameter catalog for persistence 

1611 

1612 Parameters 

1613 ---------- 

1614 parSchema: `lsst.afw.table.Schema` 

1615 Parameter catalog schema 

1616 pars: `numpy.ndarray` 

1617 FGCM parameters to put into parCat 

1618 parSuperStarFlat: `numpy.array` 

1619 FGCM superstar flat array to put into parCat 

1620 lutFilterNameString: `str` 

1621 Combined string of all the lutFilterNames 

1622 fitBandString: `str` 

1623 Combined string of all the fitBands 

1624 

1625 Returns 

1626 ------- 

1627 parCat: `afwTable.BasicCatalog` 

1628 Atmosphere and instrumental model parameter catalog for persistence 

1629 """ 

1630 

1631 parCat = afwTable.BaseCatalog(parSchema) 

1632 parCat.reserve(1) 

1633 

1634 # The parameter catalog just has one row, with many columns for all the 

1635 # atmosphere and instrument fit parameters 

1636 rec = parCat.addNew() 

1637 

1638 # info section 

1639 rec['nCcd'] = parInfo['NCCD'] 

1640 rec['lutFilterNames'] = lutFilterNameString 

1641 rec['fitBands'] = fitBandString 

1642 # note these are not currently supported here. 

1643 rec['hasExternalPwv'] = 0 

1644 rec['hasExternalTau'] = 0 

1645 

1646 # parameter section 

1647 

1648 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1649 

1650 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1651 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1652 'parQeSysIntercept', 'compQeSysSlope', 

1653 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1654 'parFilterOffset', 'parFilterOffsetFitFlag', 

1655 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1656 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1657 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1658 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1659 'compModelErrSkyPivot', 'compModelErrPars', 

1660 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1661 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope', 

1662 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1663 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper', 

1664 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap', 

1665 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap'] 

1666 

1667 for scalarName in scalarNames: 

1668 rec[scalarName] = pars[scalarName.upper()] 

1669 

1670 for arrName in arrNames: 

1671 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1672 

1673 # superstar section 

1674 rec['superstarSize'][:] = parSuperStarFlat.shape 

1675 rec['superstar'][:] = parSuperStarFlat.ravel() 

1676 

1677 return parCat 

1678 

1679 def _makeFlagStarSchema(self): 

1680 """ 

1681 Make the flagged-stars schema 

1682 

1683 Returns 

1684 ------- 

1685 flagStarSchema: `lsst.afw.table.Schema` 

1686 """ 

1687 

1688 flagStarSchema = afwTable.Schema() 

1689 

1690 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1691 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1692 

1693 return flagStarSchema 

1694 

1695 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1696 """ 

1697 Make the flagged star catalog for persistence 

1698 

1699 Parameters 

1700 ---------- 

1701 flagStarSchema: `lsst.afw.table.Schema` 

1702 Flagged star schema 

1703 flagStarStruct: `numpy.ndarray` 

1704 Flagged star structure from fgcm 

1705 

1706 Returns 

1707 ------- 

1708 flagStarCat: `lsst.afw.table.BaseCatalog` 

1709 Flagged star catalog for persistence 

1710 """ 

1711 

1712 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1713 flagStarCat.resize(flagStarStruct.size) 

1714 

1715 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1716 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1717 

1718 return flagStarCat