Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import sys 

37import traceback 

38import copy 

39 

40import numpy as np 

41 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45import lsst.afw.table as afwTable 

46from lsst.utils.timer import timeMethod 

47 

48from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

49from .utilities import extractReferenceMags 

50from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat 

51from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

52from .sedterms import SedboundarytermDict, SedtermDict 

53from .utilities import lookupStaticCalibrations 

54 

55import fgcm 

56 

57__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner'] 

58 

59MULTIPLE_CYCLES_MAX = 10 

60 

61 

62class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

63 dimensions=("instrument",), 

64 defaultTemplates={"previousCycleNumber": "-1", 

65 "cycleNumber": "0"}): 

66 camera = connectionTypes.PrerequisiteInput( 

67 doc="Camera instrument", 

68 name="camera", 

69 storageClass="Camera", 

70 dimensions=("instrument",), 

71 lookupFunction=lookupStaticCalibrations, 

72 isCalibration=True, 

73 ) 

74 

75 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

76 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

77 "chromatic corrections."), 

78 name="fgcmLookUpTable", 

79 storageClass="Catalog", 

80 dimensions=("instrument",), 

81 deferLoad=True, 

82 ) 

83 

84 fgcmVisitCatalog = connectionTypes.Input( 

85 doc="Catalog of visit information for fgcm", 

86 name="fgcmVisitCatalog", 

87 storageClass="Catalog", 

88 dimensions=("instrument",), 

89 deferLoad=True, 

90 ) 

91 

92 fgcmStarObservations = connectionTypes.Input( 

93 doc="Catalog of star observations for fgcm", 

94 name="fgcmStarObservations", 

95 storageClass="Catalog", 

96 dimensions=("instrument",), 

97 deferLoad=True, 

98 ) 

99 

100 fgcmStarIds = connectionTypes.Input( 

101 doc="Catalog of fgcm calibration star IDs", 

102 name="fgcmStarIds", 

103 storageClass="Catalog", 

104 dimensions=("instrument",), 

105 deferLoad=True, 

106 ) 

107 

108 fgcmStarIndices = connectionTypes.Input( 

109 doc="Catalog of fgcm calibration star indices", 

110 name="fgcmStarIndices", 

111 storageClass="Catalog", 

112 dimensions=("instrument",), 

113 deferLoad=True, 

114 ) 

115 

116 fgcmReferenceStars = connectionTypes.Input( 

117 doc="Catalog of fgcm-matched reference stars", 

118 name="fgcmReferenceStars", 

119 storageClass="Catalog", 

120 dimensions=("instrument",), 

121 deferLoad=True, 

122 ) 

123 

124 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput( 

125 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle", 

126 name="fgcmFlaggedStars{previousCycleNumber}", 

127 storageClass="Catalog", 

128 dimensions=("instrument",), 

129 deferLoad=True, 

130 ) 

131 

132 fgcmFitParametersInput = connectionTypes.PrerequisiteInput( 

133 doc="Catalog of fgcm fit parameters from previous fit cycle", 

134 name="fgcmFitParameters{previousCycleNumber}", 

135 storageClass="Catalog", 

136 dimensions=("instrument",), 

137 deferLoad=True, 

138 ) 

139 

140 fgcmFitParameters = connectionTypes.Output( 

141 doc="Catalog of fgcm fit parameters from current fit cycle", 

142 name="fgcmFitParameters{cycleNumber}", 

143 storageClass="Catalog", 

144 dimensions=("instrument",), 

145 ) 

146 

147 fgcmFlaggedStars = connectionTypes.Output( 

148 doc="Catalog of flagged stars for fgcm calibration from current fit cycle", 

149 name="fgcmFlaggedStars{cycleNumber}", 

150 storageClass="Catalog", 

151 dimensions=("instrument",), 

152 ) 

153 

154 fgcmZeropoints = connectionTypes.Output( 

155 doc="Catalog of fgcm zeropoint data from current fit cycle", 

156 name="fgcmZeropoints{cycleNumber}", 

157 storageClass="Catalog", 

158 dimensions=("instrument",), 

159 ) 

160 

161 fgcmAtmosphereParameters = connectionTypes.Output( 

162 doc="Catalog of atmospheric fit parameters from current fit cycle", 

163 name="fgcmAtmosphereParameters{cycleNumber}", 

164 storageClass="Catalog", 

165 dimensions=("instrument",), 

166 ) 

167 

168 fgcmStandardStars = connectionTypes.Output( 

169 doc="Catalog of standard star magnitudes from current fit cycle", 

170 name="fgcmStandardStars{cycleNumber}", 

171 storageClass="SimpleCatalog", 

172 dimensions=("instrument",), 

173 ) 

174 

175 # Add connections for running multiple cycles 

176 # This uses vars() to alter the class __dict__ to programmatically 

177 # write many similar outputs. 

178 for cycle in range(MULTIPLE_CYCLES_MAX): 

179 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output( 

180 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}", 

181 name=f"fgcmFitParameters{cycle}", 

182 storageClass="Catalog", 

183 dimensions=("instrument",), 

184 ) 

185 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output( 

186 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}", 

187 name=f"fgcmFlaggedStars{cycle}", 

188 storageClass="Catalog", 

189 dimensions=("instrument",), 

190 ) 

191 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output( 

192 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}", 

193 name=f"fgcmZeropoints{cycle}", 

194 storageClass="Catalog", 

195 dimensions=("instrument",), 

196 ) 

197 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output( 

198 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}", 

199 name=f"fgcmAtmosphereParameters{cycle}", 

200 storageClass="Catalog", 

201 dimensions=("instrument",), 

202 ) 

203 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output( 

204 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}", 

205 name=f"fgcmStandardStars{cycle}", 

206 storageClass="SimpleCatalog", 

207 dimensions=("instrument",), 

208 ) 

209 

210 def __init__(self, *, config=None): 

211 super().__init__(config=config) 

212 

213 if not config.doReferenceCalibration: 

214 self.inputs.remove("fgcmReferenceStars") 

215 

216 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

217 raise ValueError("cycleNumber must be of integer format") 

218 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

219 raise ValueError("previousCycleNumber must be of integer format") 

220 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

221 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

222 

223 if int(config.connections.cycleNumber) == 0: 

224 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput") 

225 self.prerequisiteInputs.remove("fgcmFitParametersInput") 

226 

227 if not self.config.doMultipleCycles: 

228 # Single-cycle run 

229 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle: 

230 self.outputs.remove("fgcmStandardStars") 

231 

232 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle: 

233 self.outputs.remove("fgcmZeropoints") 

234 self.outputs.remove("fgcmAtmosphereParameters") 

235 

236 # Remove all multiple cycle outputs 

237 for cycle in range(0, MULTIPLE_CYCLES_MAX): 

238 self.outputs.remove(f"fgcmFitParameters{cycle}") 

239 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

240 self.outputs.remove(f"fgcmZeropoints{cycle}") 

241 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

242 self.outputs.remove(f"fgcmStandardStars{cycle}") 

243 

244 else: 

245 # Multiple-cycle run 

246 # Remove single-cycle outputs 

247 self.outputs.remove("fgcmFitParameters") 

248 self.outputs.remove("fgcmFlaggedStars") 

249 self.outputs.remove("fgcmZeropoints") 

250 self.outputs.remove("fgcmAtmosphereParameters") 

251 self.outputs.remove("fgcmStandardStars") 

252 

253 # Remove outputs from cycles that are not used 

254 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1, 

255 MULTIPLE_CYCLES_MAX): 

256 self.outputs.remove(f"fgcmFitParameters{cycle}") 

257 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

258 self.outputs.remove(f"fgcmZeropoints{cycle}") 

259 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

260 self.outputs.remove(f"fgcmStandardStars{cycle}") 

261 

262 # Remove non-final-cycle outputs if necessary 

263 for cycle in range(self.config.multipleCyclesFinalCycleNumber): 

264 if not self.config.outputZeropointsBeforeFinalCycle: 

265 self.outputs.remove(f"fgcmZeropoints{cycle}") 

266 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

267 if not self.config.outputStandardsBeforeFinalCycle: 

268 self.outputs.remove(f"fgcmStandardStars{cycle}") 

269 

270 

271class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

272 pipelineConnections=FgcmFitCycleConnections): 

273 """Config for FgcmFitCycle""" 

274 

275 doMultipleCycles = pexConfig.Field( 

276 doc="Run multiple fit cycles in one task", 

277 dtype=bool, 

278 default=False, 

279 ) 

280 multipleCyclesFinalCycleNumber = pexConfig.RangeField( 

281 doc=("Final cycle number in multiple cycle mode. The initial cycle " 

282 "is 0, with limited parameters fit. The next cycle is 1 with " 

283 "full parameter fit. The final cycle is a clean-up with no " 

284 "parameters fit. There will be a total of " 

285 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final " 

286 "cycle number cannot be less than 2."), 

287 dtype=int, 

288 default=5, 

289 min=2, 

290 max=MULTIPLE_CYCLES_MAX, 

291 inclusiveMax=True, 

292 ) 

293 bands = pexConfig.ListField( 

294 doc="Bands to run calibration", 

295 dtype=str, 

296 default=[], 

297 ) 

298 fitBands = pexConfig.ListField( 

299 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

300 "the atmosphere constrained from the 'fitBands' on the same night. " 

301 "Must be a subset of `config.bands`"), 

302 dtype=str, 

303 default=[], 

304 ) 

305 requiredBands = pexConfig.ListField( 

306 doc=("Bands that are required for a star to be considered a calibration star. " 

307 "Must be a subset of `config.bands`"), 

308 dtype=str, 

309 default=[], 

310 ) 

311 # The following config will not be necessary after Gen2 retirement. 

312 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which 

313 # is easiest to access in the config file. 

314 physicalFilterMap = pexConfig.DictField( 

315 doc="Mapping from 'physicalFilter' to band.", 

316 keytype=str, 

317 itemtype=str, 

318 default={}, 

319 ) 

320 doReferenceCalibration = pexConfig.Field( 

321 doc="Use reference catalog as additional constraint on calibration", 

322 dtype=bool, 

323 default=True, 

324 ) 

325 refStarSnMin = pexConfig.Field( 

326 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

327 dtype=float, 

328 default=50.0, 

329 ) 

330 refStarOutlierNSig = pexConfig.Field( 

331 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

332 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

333 dtype=float, 

334 default=4.0, 

335 ) 

336 applyRefStarColorCuts = pexConfig.Field( 

337 doc="Apply color cuts to reference stars?", 

338 dtype=bool, 

339 default=True, 

340 ) 

341 nCore = pexConfig.Field( 

342 doc="Number of cores to use", 

343 dtype=int, 

344 default=4, 

345 ) 

346 nStarPerRun = pexConfig.Field( 

347 doc="Number of stars to run in each chunk", 

348 dtype=int, 

349 default=200000, 

350 ) 

351 nExpPerRun = pexConfig.Field( 

352 doc="Number of exposures to run in each chunk", 

353 dtype=int, 

354 default=1000, 

355 ) 

356 reserveFraction = pexConfig.Field( 

357 doc="Fraction of stars to reserve for testing", 

358 dtype=float, 

359 default=0.1, 

360 ) 

361 freezeStdAtmosphere = pexConfig.Field( 

362 doc="Freeze atmosphere parameters to standard (for testing)", 

363 dtype=bool, 

364 default=False, 

365 ) 

366 precomputeSuperStarInitialCycle = pexConfig.Field( 

367 doc="Precompute superstar flat for initial cycle", 

368 dtype=bool, 

369 default=False, 

370 ) 

371 superStarSubCcdDict = pexConfig.DictField( 

372 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

373 "Must have one entry per band."), 

374 keytype=str, 

375 itemtype=bool, 

376 default={}, 

377 ) 

378 superStarSubCcdChebyshevOrder = pexConfig.Field( 

379 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

380 "Global default is first-order polynomials, and should be overridden " 

381 "on a camera-by-camera basis depending on the ISR."), 

382 dtype=int, 

383 default=1, 

384 ) 

385 superStarSubCcdTriangular = pexConfig.Field( 

386 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

387 "suppress high-order cross terms?"), 

388 dtype=bool, 

389 default=False, 

390 ) 

391 superStarSigmaClip = pexConfig.Field( 

392 doc="Number of sigma to clip outliers when selecting for superstar flats", 

393 dtype=float, 

394 default=5.0, 

395 ) 

396 focalPlaneSigmaClip = pexConfig.Field( 

397 doc="Number of sigma to clip outliers per focal-plane.", 

398 dtype=float, 

399 default=4.0, 

400 ) 

401 ccdGraySubCcdDict = pexConfig.DictField( 

402 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

403 "('ccd gray') on a sub-ccd scale."), 

404 keytype=str, 

405 itemtype=bool, 

406 default={}, 

407 ) 

408 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

409 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

410 dtype=int, 

411 default=1, 

412 ) 

413 ccdGraySubCcdTriangular = pexConfig.Field( 

414 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

415 "suppress high-order cross terms?"), 

416 dtype=bool, 

417 default=True, 

418 ) 

419 ccdGrayFocalPlaneDict = pexConfig.DictField( 

420 doc=("Per-band specification on whether to compute focal-plane residual " 

421 "('ccd gray') corrections."), 

422 keytype=str, 

423 itemtype=bool, 

424 default={}, 

425 ) 

426 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

427 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

428 "gray corrections. If there are fewer good CCDs then the gray " 

429 "correction is computed per-ccd."), 

430 dtype=int, 

431 default=1, 

432 ) 

433 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

434 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

435 dtype=int, 

436 default=3, 

437 ) 

438 cycleNumber = pexConfig.Field( 

439 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

440 "and stage of outlier rejection. See cookbook for details."), 

441 dtype=int, 

442 default=None, 

443 ) 

444 isFinalCycle = pexConfig.Field( 

445 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

446 "selection of stars and photometric exposures, and will output zeropoints " 

447 "and standard stars for use in fgcmOutputProducts"), 

448 dtype=bool, 

449 default=False, 

450 ) 

451 maxIterBeforeFinalCycle = pexConfig.Field( 

452 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

453 "will always be 0 in the final cycle for cleanup and final selection."), 

454 dtype=int, 

455 default=50, 

456 ) 

457 deltaMagBkgOffsetPercentile = pexConfig.Field( 

458 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

459 "offset from local background subtraction."), 

460 dtype=float, 

461 default=0.25, 

462 ) 

463 deltaMagBkgPerCcd = pexConfig.Field( 

464 doc=("Compute net offset from local background subtraction per-ccd? " 

465 "Otherwise, use computation per visit."), 

466 dtype=bool, 

467 default=False, 

468 ) 

469 utBoundary = pexConfig.Field( 

470 doc="Boundary (in UTC) from day-to-day", 

471 dtype=float, 

472 default=None, 

473 ) 

474 washMjds = pexConfig.ListField( 

475 doc="Mirror wash MJDs", 

476 dtype=float, 

477 default=(0.0,), 

478 ) 

479 epochMjds = pexConfig.ListField( 

480 doc="Epoch boundaries in MJD", 

481 dtype=float, 

482 default=(0.0,), 

483 ) 

484 minObsPerBand = pexConfig.Field( 

485 doc="Minimum good observations per band", 

486 dtype=int, 

487 default=2, 

488 ) 

489 # TODO: When DM-16511 is done, it will be possible to get the 

490 # telescope latitude directly from the camera. 

491 latitude = pexConfig.Field( 

492 doc="Observatory latitude", 

493 dtype=float, 

494 default=None, 

495 ) 

496 brightObsGrayMax = pexConfig.Field( 

497 doc="Maximum gray extinction to be considered bright observation", 

498 dtype=float, 

499 default=0.15, 

500 ) 

501 minStarPerCcd = pexConfig.Field( 

502 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

503 "CCDs with fewer stars will have their calibration estimated from other " 

504 "CCDs in the same visit, with zeropoint error increased accordingly."), 

505 dtype=int, 

506 default=5, 

507 ) 

508 minCcdPerExp = pexConfig.Field( 

509 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

510 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

511 dtype=int, 

512 default=5, 

513 ) 

514 maxCcdGrayErr = pexConfig.Field( 

515 doc="Maximum error on CCD gray offset to be considered photometric", 

516 dtype=float, 

517 default=0.05, 

518 ) 

519 minStarPerExp = pexConfig.Field( 

520 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

521 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

522 dtype=int, 

523 default=600, 

524 ) 

525 minExpPerNight = pexConfig.Field( 

526 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

527 dtype=int, 

528 default=10, 

529 ) 

530 expGrayInitialCut = pexConfig.Field( 

531 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

532 "observations."), 

533 dtype=float, 

534 default=-0.25, 

535 ) 

536 expGrayPhotometricCutDict = pexConfig.DictField( 

537 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

538 "('gray term') for a visit to be considered photometric. Must have one " 

539 "entry per band. Broad-band filters should be -0.05."), 

540 keytype=str, 

541 itemtype=float, 

542 default={}, 

543 ) 

544 expGrayHighCutDict = pexConfig.DictField( 

545 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

546 "('gray term') for a visit to be considered photometric. Must have one " 

547 "entry per band. Broad-band filters should be 0.2."), 

548 keytype=str, 

549 itemtype=float, 

550 default={}, 

551 ) 

552 expGrayRecoverCut = pexConfig.Field( 

553 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

554 "Visits with more gray extinction will only get CCD zeropoints if there are " 

555 "sufficient star observations (minStarPerCcd) on that CCD."), 

556 dtype=float, 

557 default=-1.0, 

558 ) 

559 expVarGrayPhotometricCutDict = pexConfig.DictField( 

560 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

561 "photometric. Must have one entry per band. Broad-band filters should be " 

562 "0.0005."), 

563 keytype=str, 

564 itemtype=float, 

565 default={}, 

566 ) 

567 expGrayErrRecoverCut = pexConfig.Field( 

568 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

569 "Visits with more gray variance will only get CCD zeropoints if there are " 

570 "sufficient star observations (minStarPerCcd) on that CCD."), 

571 dtype=float, 

572 default=0.05, 

573 ) 

574 aperCorrFitNBins = pexConfig.Field( 

575 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

576 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

577 "used if available."), 

578 dtype=int, 

579 default=10, 

580 ) 

581 aperCorrInputSlopeDict = pexConfig.DictField( 

582 doc=("Per-band specification of aperture correction input slope parameters. These " 

583 "are used on the first fit iteration, and aperture correction parameters will " 

584 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

585 "to set this when there is insufficient data to fit the parameters (e.g. " 

586 "tract mode)."), 

587 keytype=str, 

588 itemtype=float, 

589 default={}, 

590 ) 

591 sedboundaryterms = pexConfig.ConfigField( 

592 doc="Mapping from bands to SED boundary term names used is sedterms.", 

593 dtype=SedboundarytermDict, 

594 ) 

595 sedterms = pexConfig.ConfigField( 

596 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

597 dtype=SedtermDict, 

598 ) 

599 sigFgcmMaxErr = pexConfig.Field( 

600 doc="Maximum mag error for fitting sigma_FGCM", 

601 dtype=float, 

602 default=0.01, 

603 ) 

604 sigFgcmMaxEGrayDict = pexConfig.DictField( 

605 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

606 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

607 "should be 0.05."), 

608 keytype=str, 

609 itemtype=float, 

610 default={}, 

611 ) 

612 ccdGrayMaxStarErr = pexConfig.Field( 

613 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

614 "computation"), 

615 dtype=float, 

616 default=0.10, 

617 ) 

618 approxThroughputDict = pexConfig.DictField( 

619 doc=("Per-band specification of the approximate overall throughput at the start of " 

620 "calibration observations. Must have one entry per band. Typically should " 

621 "be 1.0."), 

622 keytype=str, 

623 itemtype=float, 

624 default={}, 

625 ) 

626 sigmaCalRange = pexConfig.ListField( 

627 doc="Allowed range for systematic error floor estimation", 

628 dtype=float, 

629 default=(0.001, 0.003), 

630 ) 

631 sigmaCalFitPercentile = pexConfig.ListField( 

632 doc="Magnitude percentile range to fit systematic error floor", 

633 dtype=float, 

634 default=(0.05, 0.15), 

635 ) 

636 sigmaCalPlotPercentile = pexConfig.ListField( 

637 doc="Magnitude percentile range to plot systematic error floor", 

638 dtype=float, 

639 default=(0.05, 0.95), 

640 ) 

641 sigma0Phot = pexConfig.Field( 

642 doc="Systematic error floor for all zeropoints", 

643 dtype=float, 

644 default=0.003, 

645 ) 

646 mapLongitudeRef = pexConfig.Field( 

647 doc="Reference longitude for plotting maps", 

648 dtype=float, 

649 default=0.0, 

650 ) 

651 mapNSide = pexConfig.Field( 

652 doc="Healpix nside for plotting maps", 

653 dtype=int, 

654 default=256, 

655 ) 

656 outfileBase = pexConfig.Field( 

657 doc="Filename start for plot output files", 

658 dtype=str, 

659 default=None, 

660 ) 

661 starColorCuts = pexConfig.ListField( 

662 doc="Encoded star-color cuts (to be cleaned up)", 

663 dtype=str, 

664 default=("NO_DATA",), 

665 ) 

666 colorSplitBands = pexConfig.ListField( 

667 doc="Band names to use to split stars by color. Must have 2 entries.", 

668 dtype=str, 

669 length=2, 

670 default=('g', 'i'), 

671 ) 

672 modelMagErrors = pexConfig.Field( 

673 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

674 dtype=bool, 

675 default=True, 

676 ) 

677 useQuadraticPwv = pexConfig.Field( 

678 doc="Model PWV with a quadratic term for variation through the night?", 

679 dtype=bool, 

680 default=False, 

681 ) 

682 instrumentParsPerBand = pexConfig.Field( 

683 doc=("Model instrumental parameters per band? " 

684 "Otherwise, instrumental parameters (QE changes with time) are " 

685 "shared among all bands."), 

686 dtype=bool, 

687 default=False, 

688 ) 

689 instrumentSlopeMinDeltaT = pexConfig.Field( 

690 doc=("Minimum time change (in days) between observations to use in constraining " 

691 "instrument slope."), 

692 dtype=float, 

693 default=20.0, 

694 ) 

695 fitMirrorChromaticity = pexConfig.Field( 

696 doc="Fit (intraband) mirror chromatic term?", 

697 dtype=bool, 

698 default=False, 

699 ) 

700 coatingMjds = pexConfig.ListField( 

701 doc="Mirror coating dates in MJD", 

702 dtype=float, 

703 default=(0.0,), 

704 ) 

705 outputStandardsBeforeFinalCycle = pexConfig.Field( 

706 doc="Output standard stars prior to final cycle? Used in debugging.", 

707 dtype=bool, 

708 default=False, 

709 ) 

710 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

711 doc="Output standard stars prior to final cycle? Used in debugging.", 

712 dtype=bool, 

713 default=False, 

714 ) 

715 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

716 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

717 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

718 keytype=str, 

719 itemtype=bool, 

720 default={}, 

721 ) 

722 autoPhotometricCutNSig = pexConfig.Field( 

723 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

724 "Cut is based on exposure gray width (per band), unless " 

725 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

726 "repeatability is used (also per band)."), 

727 dtype=float, 

728 default=3.0, 

729 ) 

730 autoHighCutNSig = pexConfig.Field( 

731 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

732 "Cut is based on exposure gray width (per band), unless " 

733 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

734 "repeatability is used (also per band)."), 

735 dtype=float, 

736 default=4.0, 

737 ) 

738 quietMode = pexConfig.Field( 

739 doc="Be less verbose with logging.", 

740 dtype=bool, 

741 default=False, 

742 ) 

743 doPlots = pexConfig.Field( 

744 doc="Make fgcm QA plots.", 

745 dtype=bool, 

746 default=True, 

747 ) 

748 randomSeed = pexConfig.Field( 

749 doc="Random seed for fgcm for consistency in tests.", 

750 dtype=int, 

751 default=None, 

752 optional=True, 

753 ) 

754 

755 def validate(self): 

756 super().validate() 

757 

758 if self.connections.previousCycleNumber != str(self.cycleNumber - 1): 

759 msg = "cycleNumber in template must be connections.previousCycleNumber + 1" 

760 raise RuntimeError(msg) 

761 if self.connections.cycleNumber != str(self.cycleNumber): 

762 msg = "cycleNumber in template must be equal to connections.cycleNumber" 

763 raise RuntimeError(msg) 

764 

765 for band in self.fitBands: 

766 if band not in self.bands: 

767 msg = 'fitBand %s not in bands' % (band) 

768 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

769 for band in self.requiredBands: 

770 if band not in self.bands: 

771 msg = 'requiredBand %s not in bands' % (band) 

772 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

773 for band in self.colorSplitBands: 

774 if band not in self.bands: 

775 msg = 'colorSplitBand %s not in bands' % (band) 

776 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

777 for band in self.bands: 

778 if band not in self.superStarSubCcdDict: 

779 msg = 'band %s not in superStarSubCcdDict' % (band) 

780 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

781 self, msg) 

782 if band not in self.ccdGraySubCcdDict: 

783 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

784 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

785 self, msg) 

786 if band not in self.expGrayPhotometricCutDict: 

787 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

788 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

789 self, msg) 

790 if band not in self.expGrayHighCutDict: 

791 msg = 'band %s not in expGrayHighCutDict' % (band) 

792 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

793 self, msg) 

794 if band not in self.expVarGrayPhotometricCutDict: 

795 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

796 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

797 self, msg) 

798 if band not in self.sigFgcmMaxEGrayDict: 

799 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

800 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

801 self, msg) 

802 if band not in self.approxThroughputDict: 

803 msg = 'band %s not in approxThroughputDict' % (band) 

804 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

805 self, msg) 

806 if band not in self.useRepeatabilityForExpGrayCutsDict: 

807 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

808 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

809 self, msg) 

810 

811 

812class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner): 

813 """Subclass of TaskRunner for fgcmFitCycleTask 

814 

815 fgcmFitCycleTask.run() takes one argument, the butler, and uses 

816 stars and visits previously extracted from dataRefs by 

817 fgcmBuildStars. 

818 This Runner does not perform any dataRef parallelization, but the FGCM 

819 code called by the Task uses python multiprocessing (see the "ncores" 

820 config option). 

821 """ 

822 

823 @staticmethod 

824 def getTargetList(parsedCmd): 

825 """ 

826 Return a list with one element, the butler. 

827 """ 

828 return [parsedCmd.butler] 

829 

830 def __call__(self, butler): 

831 """ 

832 Parameters 

833 ---------- 

834 butler: `lsst.daf.persistence.Butler` 

835 

836 Returns 

837 ------- 

838 exitStatus: `list` with `pipeBase.Struct` 

839 exitStatus (0: success; 1: failure) 

840 """ 

841 

842 task = self.TaskClass(config=self.config, log=self.log) 

843 

844 exitStatus = 0 

845 if self.doRaise: 

846 task.runDataRef(butler) 

847 else: 

848 try: 

849 task.runDataRef(butler) 

850 except Exception as e: 

851 exitStatus = 1 

852 task.log.fatal("Failed: %s" % e) 

853 if not isinstance(e, pipeBase.TaskError): 

854 traceback.print_exc(file=sys.stderr) 

855 

856 task.writeMetadata(butler) 

857 

858 # The task does not return any results: 

859 return [pipeBase.Struct(exitStatus=exitStatus)] 

860 

861 def run(self, parsedCmd): 

862 """ 

863 Run the task, with no multiprocessing 

864 

865 Parameters 

866 ---------- 

867 parsedCmd: ArgumentParser parsed command line 

868 """ 

869 

870 resultList = [] 

871 

872 if self.precall(parsedCmd): 

873 targetList = self.getTargetList(parsedCmd) 

874 # make sure that we only get 1 

875 resultList = self(targetList[0]) 

876 

877 return resultList 

878 

879 

880class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

881 """ 

882 Run Single fit cycle for FGCM global calibration 

883 """ 

884 

885 ConfigClass = FgcmFitCycleConfig 

886 RunnerClass = FgcmFitCycleRunner 

887 _DefaultName = "fgcmFitCycle" 

888 

889 def __init__(self, butler=None, initInputs=None, **kwargs): 

890 super().__init__(**kwargs) 

891 

892 # no saving of metadata for now 

893 def _getMetadataName(self): 

894 return None 

895 

896 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

897 camera = butlerQC.get(inputRefs.camera) 

898 

899 dataRefDict = {} 

900 

901 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

902 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

903 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations) 

904 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds) 

905 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices) 

906 if self.config.doReferenceCalibration: 

907 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars) 

908 if self.config.cycleNumber > 0: 

909 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput) 

910 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput) 

911 

912 fgcmDatasetDict = None 

913 if self.config.doMultipleCycles: 

914 # Run multiple cycles at once. 

915 config = copy.copy(self.config) 

916 config.update(cycleNumber=0) 

917 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1): 

918 if cycle == self.config.multipleCyclesFinalCycleNumber: 

919 config.update(isFinalCycle=True) 

920 

921 if cycle > 0: 

922 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars'] 

923 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters'] 

924 

925 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config) 

926 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], 

927 getattr(outputRefs, f'fgcmFitParameters{cycle}')) 

928 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], 

929 getattr(outputRefs, f'fgcmFlaggedStars{cycle}')) 

930 if self.outputZeropoints: 

931 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], 

932 getattr(outputRefs, f'fgcmZeropoints{cycle}')) 

933 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 

934 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}')) 

935 if self.outputStandards: 

936 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], 

937 getattr(outputRefs, f'fgcmStandardStars{cycle}')) 

938 else: 

939 # Run a single cycle 

940 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict) 

941 

942 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters) 

943 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars) 

944 if self.outputZeropoints: 

945 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints) 

946 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters) 

947 if self.outputStandards: 

948 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars) 

949 

950 @timeMethod 

951 def runDataRef(self, butler): 

952 """ 

953 Run a single fit cycle for FGCM 

954 

955 Parameters 

956 ---------- 

957 butler: `lsst.daf.persistence.Butler` 

958 """ 

959 self._checkDatasetsExist(butler) 

960 

961 dataRefDict = {} 

962 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

963 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

964 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations') 

965 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds') 

966 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices') 

967 if self.config.doReferenceCalibration: 

968 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars') 

969 if self.config.cycleNumber > 0: 

970 lastCycle = self.config.cycleNumber - 1 

971 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars', 

972 fgcmcycle=lastCycle) 

973 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters', 

974 fgcmcycle=lastCycle) 

975 

976 camera = butler.get('camera') 

977 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict) 

978 

979 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters', 

980 fgcmcycle=self.config.cycleNumber) 

981 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars', 

982 fgcmcycle=self.config.cycleNumber) 

983 if self.outputZeropoints: 

984 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints', 

985 fgcmcycle=self.config.cycleNumber) 

986 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters', 

987 fgcmcycle=self.config.cycleNumber) 

988 if self.outputStandards: 

989 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars', 

990 fgcmcycle=self.config.cycleNumber) 

991 

992 def writeConfig(self, butler, clobber=False, doBackup=True): 

993 """Write the configuration used for processing the data, or check that an existing 

994 one is equal to the new one if present. This is an override of the regular 

995 version from pipe_base that knows about fgcmcycle. 

996 

997 Parameters 

998 ---------- 

999 butler : `lsst.daf.persistence.Butler` 

1000 Data butler used to write the config. The config is written to dataset type 

1001 `CmdLineTask._getConfigName`. 

1002 clobber : `bool`, optional 

1003 A boolean flag that controls what happens if a config already has been saved: 

1004 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 

1005 - `False`: raise `TaskError` if this config does not match the existing config. 

1006 doBackup : `bool`, optional 

1007 Set to `True` to backup the config files if clobbering. 

1008 """ 

1009 configName = self._getConfigName() 

1010 if configName is None: 

1011 return 

1012 if clobber: 

1013 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber) 

1014 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber): 

1015 # this may be subject to a race condition; see #2789 

1016 try: 

1017 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber) 

1018 except Exception as exc: 

1019 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" % 

1020 (configName, exc)) 

1021 

1022 def logConfigMismatch(msg): 

1023 self.log.fatal("Comparing configuration: %s", msg) 

1024 

1025 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

1026 raise pipeBase.TaskError( 

1027 f"Config does not match existing task config {configName!r} on disk; tasks configurations" 

1028 " must be consistent within the same output repo (override with --clobber-config)") 

1029 else: 

1030 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber) 

1031 

1032 def _fgcmFitCycle(self, camera, dataRefDict, config=None): 

1033 """ 

1034 Run the fit cycle 

1035 

1036 Parameters 

1037 ---------- 

1038 camera : `lsst.afw.cameraGeom.Camera` 

1039 dataRefDict : `dict` 

1040 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1041 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1042 dataRef dictionary with keys: 

1043 

1044 ``"fgcmLookUpTable"`` 

1045 dataRef for the FGCM look-up table. 

1046 ``"fgcmVisitCatalog"`` 

1047 dataRef for visit summary catalog. 

1048 ``"fgcmStarObservations"`` 

1049 dataRef for star observation catalog. 

1050 ``"fgcmStarIds"`` 

1051 dataRef for star id catalog. 

1052 ``"fgcmStarIndices"`` 

1053 dataRef for star index catalog. 

1054 ``"fgcmReferenceStars"`` 

1055 dataRef for matched reference star catalog. 

1056 ``"fgcmFlaggedStars"`` 

1057 dataRef for flagged star catalog. 

1058 ``"fgcmFitParameters"`` 

1059 dataRef for fit parameter catalog. 

1060 config : `lsst.pex.config.Config`, optional 

1061 Configuration to use to override self.config. 

1062 

1063 Returns 

1064 ------- 

1065 fgcmDatasetDict : `dict` 

1066 Dictionary of datasets to persist. 

1067 """ 

1068 if config is not None: 

1069 _config = config 

1070 else: 

1071 _config = self.config 

1072 

1073 # Set defaults on whether to output standards and zeropoints 

1074 self.maxIter = _config.maxIterBeforeFinalCycle 

1075 self.outputStandards = _config.outputStandardsBeforeFinalCycle 

1076 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle 

1077 self.resetFitParameters = True 

1078 

1079 if _config.isFinalCycle: 

1080 # This is the final fit cycle, so we do not want to reset fit 

1081 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

1082 # and we always want to output standards and zeropoints 

1083 self.maxIter = 0 

1084 self.outputStandards = True 

1085 self.outputZeropoints = True 

1086 self.resetFitParameters = False 

1087 

1088 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1089 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

1090 dict(_config.physicalFilterMap)) 

1091 del lutCat 

1092 

1093 configDict = makeConfigDict(_config, self.log, camera, 

1094 self.maxIter, self.resetFitParameters, 

1095 self.outputZeropoints, 

1096 lutIndexVals[0]['FILTERNAMES']) 

1097 

1098 # next we need the exposure/visit information 

1099 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

1100 fgcmExpInfo = translateVisitCatalog(visitCat) 

1101 del visitCat 

1102 

1103 # Use the first orientation. 

1104 # TODO: DM-21215 will generalize to arbitrary camera orientations 

1105 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

1106 

1107 noFitsDict = {'lutIndex': lutIndexVals, 

1108 'lutStd': lutStd, 

1109 'expInfo': fgcmExpInfo, 

1110 'ccdOffsets': ccdOffsets} 

1111 

1112 # set up the fitter object 

1113 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

1114 noFitsDict=noFitsDict, noOutput=True) 

1115 

1116 # create the parameter object 

1117 if (fgcmFitCycle.initialCycle): 

1118 # cycle = 0, initial cycle 

1119 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

1120 fgcmLut, 

1121 fgcmExpInfo) 

1122 else: 

1123 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog): 

1124 parCat = dataRefDict['fgcmFitParameters'] 

1125 else: 

1126 parCat = dataRefDict['fgcmFitParameters'].get() 

1127 inParInfo, inParams, inSuperStar = self._loadParameters(parCat) 

1128 del parCat 

1129 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

1130 fgcmExpInfo, 

1131 inParInfo, 

1132 inParams, 

1133 inSuperStar) 

1134 

1135 # set up the stars... 

1136 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

1137 

1138 starObs = dataRefDict['fgcmStarObservations'].get() 

1139 starIds = dataRefDict['fgcmStarIds'].get() 

1140 starIndices = dataRefDict['fgcmStarIndices'].get() 

1141 

1142 # grab the flagged stars if available 

1143 if 'fgcmFlaggedStars' in dataRefDict: 

1144 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog): 

1145 flaggedStars = dataRefDict['fgcmFlaggedStars'] 

1146 else: 

1147 flaggedStars = dataRefDict['fgcmFlaggedStars'].get() 

1148 flagId = flaggedStars['objId'][:] 

1149 flagFlag = flaggedStars['objFlag'][:] 

1150 else: 

1151 flaggedStars = None 

1152 flagId = None 

1153 flagFlag = None 

1154 

1155 if _config.doReferenceCalibration: 

1156 refStars = dataRefDict['fgcmReferenceStars'].get() 

1157 

1158 refMag, refMagErr = extractReferenceMags(refStars, 

1159 _config.bands, 

1160 _config.physicalFilterMap) 

1161 refId = refStars['fgcm_id'][:] 

1162 else: 

1163 refStars = None 

1164 refId = None 

1165 refMag = None 

1166 refMagErr = None 

1167 

1168 # match star observations to visits 

1169 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

1170 # actually be transferred into fgcm using the indexing below. 

1171 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

1172 

1173 # The fgcmStars.loadStars method will copy all the star information into 

1174 # special shared memory objects that will not blow up the memory usage when 

1175 # used with python multiprocessing. Once all the numbers are copied, 

1176 # it is necessary to release all references to the objects that previously 

1177 # stored the data to ensure that the garbage collector can clear the memory, 

1178 # and ensure that this memory is not copied when multiprocessing kicks in. 

1179 

1180 # We determine the conversion from the native units (typically radians) to 

1181 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

1182 # numpy arrays rather than Angles, which would we approximately 600x slower. 

1183 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

1184 

1185 fgcmStars.loadStars(fgcmPars, 

1186 starObs['visit'][starIndices['obsIndex']], 

1187 starObs['ccd'][starIndices['obsIndex']], 

1188 starObs['ra'][starIndices['obsIndex']] * conv, 

1189 starObs['dec'][starIndices['obsIndex']] * conv, 

1190 starObs['instMag'][starIndices['obsIndex']], 

1191 starObs['instMagErr'][starIndices['obsIndex']], 

1192 fgcmExpInfo['FILTERNAME'][visitIndex], 

1193 starIds['fgcm_id'][:], 

1194 starIds['ra'][:], 

1195 starIds['dec'][:], 

1196 starIds['obsArrIndex'][:], 

1197 starIds['nObs'][:], 

1198 obsX=starObs['x'][starIndices['obsIndex']], 

1199 obsY=starObs['y'][starIndices['obsIndex']], 

1200 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']], 

1201 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

1202 refID=refId, 

1203 refMag=refMag, 

1204 refMagErr=refMagErr, 

1205 flagID=flagId, 

1206 flagFlag=flagFlag, 

1207 computeNobs=True) 

1208 

1209 # Release all references to temporary objects holding star data (see above) 

1210 del starObs 

1211 del starIds 

1212 del starIndices 

1213 del flagId 

1214 del flagFlag 

1215 del flaggedStars 

1216 del refStars 

1217 del refId 

1218 del refMag 

1219 del refMagErr 

1220 

1221 # and set the bits in the cycle object 

1222 fgcmFitCycle.setLUT(fgcmLut) 

1223 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

1224 fgcmFitCycle.setPars(fgcmPars) 

1225 

1226 # finish the setup 

1227 fgcmFitCycle.finishSetup() 

1228 

1229 # and run 

1230 fgcmFitCycle.run() 

1231 

1232 ################## 

1233 # Persistance 

1234 ################## 

1235 

1236 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle) 

1237 

1238 # Output the config for the next cycle 

1239 # We need to make a copy since the input one has been frozen 

1240 

1241 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

1242 i, b in enumerate(_config.bands)} 

1243 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

1244 i, band in enumerate(_config.bands)} 

1245 

1246 outConfig = copy.copy(_config) 

1247 outConfig.update(cycleNumber=(_config.cycleNumber + 1), 

1248 precomputeSuperStarInitialCycle=False, 

1249 freezeStdAtmosphere=False, 

1250 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

1251 expGrayHighCutDict=updatedHighCutDict) 

1252 

1253 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber), 

1254 cycleNumber=str(_config.cycleNumber + 1)) 

1255 

1256 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

1257 outConfig.cycleNumber) 

1258 outConfig.save(configFileName) 

1259 

1260 if _config.isFinalCycle == 1: 

1261 # We are done, ready to output products 

1262 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

1263 else: 

1264 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

1265 self.log.info("Be sure to look at:") 

1266 self.log.info(" config.expGrayPhotometricCut") 

1267 self.log.info(" config.expGrayHighCut") 

1268 self.log.info("If you are satisfied with the fit, please set:") 

1269 self.log.info(" config.isFinalCycle = True") 

1270 

1271 fgcmFitCycle.freeSharedMemory() 

1272 

1273 return fgcmDatasetDict, outConfig 

1274 

1275 def _checkDatasetsExist(self, butler): 

1276 """ 

1277 Check if necessary datasets exist to run fgcmFitCycle 

1278 

1279 Parameters 

1280 ---------- 

1281 butler: `lsst.daf.persistence.Butler` 

1282 

1283 Raises 

1284 ------ 

1285 RuntimeError 

1286 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 

1287 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 

1288 If cycleNumber > 0, then also checks for fgcmFitParameters, 

1289 fgcmFlaggedStars. 

1290 """ 

1291 

1292 if not butler.datasetExists('fgcmVisitCatalog'): 

1293 raise RuntimeError("Could not find fgcmVisitCatalog in repo!") 

1294 if not butler.datasetExists('fgcmStarObservations'): 

1295 raise RuntimeError("Could not find fgcmStarObservations in repo!") 

1296 if not butler.datasetExists('fgcmStarIds'): 

1297 raise RuntimeError("Could not find fgcmStarIds in repo!") 

1298 if not butler.datasetExists('fgcmStarIndices'): 

1299 raise RuntimeError("Could not find fgcmStarIndices in repo!") 

1300 if not butler.datasetExists('fgcmLookUpTable'): 

1301 raise RuntimeError("Could not find fgcmLookUpTable in repo!") 

1302 

1303 # Need additional datasets if we are not the initial cycle 

1304 if (self.config.cycleNumber > 0): 

1305 if not butler.datasetExists('fgcmFitParameters', 

1306 fgcmcycle=self.config.cycleNumber-1): 

1307 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" % 

1308 (self.config.cycleNumber-1)) 

1309 if not butler.datasetExists('fgcmFlaggedStars', 

1310 fgcmcycle=self.config.cycleNumber-1): 

1311 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" % 

1312 (self.config.cycleNumber-1)) 

1313 

1314 # And additional dataset if we want reference calibration 

1315 if self.config.doReferenceCalibration: 

1316 if not butler.datasetExists('fgcmReferenceStars'): 

1317 raise RuntimeError("Could not find fgcmReferenceStars in repo, and " 

1318 "doReferenceCalibration is True.") 

1319 

1320 def _loadParameters(self, parCat): 

1321 """ 

1322 Load FGCM parameters from a previous fit cycle 

1323 

1324 Parameters 

1325 ---------- 

1326 parCat : `lsst.afw.table.BaseCatalog` 

1327 Parameter catalog in afw table form. 

1328 

1329 Returns 

1330 ------- 

1331 inParInfo: `numpy.ndarray` 

1332 Numpy array parameter information formatted for input to fgcm 

1333 inParameters: `numpy.ndarray` 

1334 Numpy array parameter values formatted for input to fgcm 

1335 inSuperStar: `numpy.array` 

1336 Superstar flat formatted for input to fgcm 

1337 """ 

1338 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1339 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1340 

1341 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1342 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1343 (parLutFilterNames.size, )), 

1344 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1345 ('LNTAUUNIT', 'f8'), 

1346 ('LNTAUSLOPEUNIT', 'f8'), 

1347 ('ALPHAUNIT', 'f8'), 

1348 ('LNPWVUNIT', 'f8'), 

1349 ('LNPWVSLOPEUNIT', 'f8'), 

1350 ('LNPWVQUADRATICUNIT', 'f8'), 

1351 ('LNPWVGLOBALUNIT', 'f8'), 

1352 ('O3UNIT', 'f8'), 

1353 ('QESYSUNIT', 'f8'), 

1354 ('FILTEROFFSETUNIT', 'f8'), 

1355 ('HASEXTERNALPWV', 'i2'), 

1356 ('HASEXTERNALTAU', 'i2')]) 

1357 inParInfo['NCCD'] = parCat['nCcd'] 

1358 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1359 inParInfo['FITBANDS'][:] = parFitBands 

1360 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1361 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1362 

1363 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1364 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1365 ('PARLNTAUINTERCEPT', 'f8', 

1366 (parCat['parLnTauIntercept'].size, )), 

1367 ('PARLNTAUSLOPE', 'f8', 

1368 (parCat['parLnTauSlope'].size, )), 

1369 ('PARLNPWVINTERCEPT', 'f8', 

1370 (parCat['parLnPwvIntercept'].size, )), 

1371 ('PARLNPWVSLOPE', 'f8', 

1372 (parCat['parLnPwvSlope'].size, )), 

1373 ('PARLNPWVQUADRATIC', 'f8', 

1374 (parCat['parLnPwvQuadratic'].size, )), 

1375 ('PARQESYSINTERCEPT', 'f8', 

1376 (parCat['parQeSysIntercept'].size, )), 

1377 ('COMPQESYSSLOPE', 'f8', 

1378 (parCat['compQeSysSlope'].size, )), 

1379 ('PARFILTEROFFSET', 'f8', 

1380 (parCat['parFilterOffset'].size, )), 

1381 ('PARFILTEROFFSETFITFLAG', 'i2', 

1382 (parCat['parFilterOffsetFitFlag'].size, )), 

1383 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1384 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1385 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1386 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1387 ('COMPABSTHROUGHPUT', 'f8', 

1388 (parCat['compAbsThroughput'].size, )), 

1389 ('COMPREFOFFSET', 'f8', 

1390 (parCat['compRefOffset'].size, )), 

1391 ('COMPREFSIGMA', 'f8', 

1392 (parCat['compRefSigma'].size, )), 

1393 ('COMPMIRRORCHROMATICITY', 'f8', 

1394 (parCat['compMirrorChromaticity'].size, )), 

1395 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1396 (parCat['mirrorChromaticityPivot'].size, )), 

1397 ('COMPMEDIANSEDSLOPE', 'f8', 

1398 (parCat['compMedianSedSlope'].size, )), 

1399 ('COMPAPERCORRPIVOT', 'f8', 

1400 (parCat['compAperCorrPivot'].size, )), 

1401 ('COMPAPERCORRSLOPE', 'f8', 

1402 (parCat['compAperCorrSlope'].size, )), 

1403 ('COMPAPERCORRSLOPEERR', 'f8', 

1404 (parCat['compAperCorrSlopeErr'].size, )), 

1405 ('COMPAPERCORRRANGE', 'f8', 

1406 (parCat['compAperCorrRange'].size, )), 

1407 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1408 (parCat['compModelErrExptimePivot'].size, )), 

1409 ('COMPMODELERRFWHMPIVOT', 'f8', 

1410 (parCat['compModelErrFwhmPivot'].size, )), 

1411 ('COMPMODELERRSKYPIVOT', 'f8', 

1412 (parCat['compModelErrSkyPivot'].size, )), 

1413 ('COMPMODELERRPARS', 'f8', 

1414 (parCat['compModelErrPars'].size, )), 

1415 ('COMPEXPGRAY', 'f8', 

1416 (parCat['compExpGray'].size, )), 

1417 ('COMPVARGRAY', 'f8', 

1418 (parCat['compVarGray'].size, )), 

1419 ('COMPEXPDELTAMAGBKG', 'f8', 

1420 (parCat['compExpDeltaMagBkg'].size, )), 

1421 ('COMPNGOODSTARPEREXP', 'i4', 

1422 (parCat['compNGoodStarPerExp'].size, )), 

1423 ('COMPSIGFGCM', 'f8', 

1424 (parCat['compSigFgcm'].size, )), 

1425 ('COMPSIGMACAL', 'f8', 

1426 (parCat['compSigmaCal'].size, )), 

1427 ('COMPRETRIEVEDLNPWV', 'f8', 

1428 (parCat['compRetrievedLnPwv'].size, )), 

1429 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1430 (parCat['compRetrievedLnPwvRaw'].size, )), 

1431 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1432 (parCat['compRetrievedLnPwvFlag'].size, )), 

1433 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1434 (parCat['compRetrievedTauNight'].size, ))]) 

1435 

1436 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1437 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1438 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1439 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1440 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1441 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1442 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1443 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1444 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1445 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1446 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1447 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1448 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1449 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1450 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1451 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1452 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1453 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1454 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1455 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :] 

1456 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1457 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1458 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1459 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1460 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1461 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1462 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1463 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1464 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1465 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1466 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :] 

1467 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1468 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1469 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1470 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1471 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1472 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1473 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1474 

1475 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1476 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1477 

1478 return (inParInfo, inParams, inSuperStar) 

1479 

1480 def _makeFgcmOutputDatasets(self, fgcmFitCycle): 

1481 """ 

1482 Persist FGCM datasets through the butler. 

1483 

1484 Parameters 

1485 ---------- 

1486 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1487 Fgcm Fit cycle object 

1488 """ 

1489 fgcmDatasetDict = {} 

1490 

1491 # Save the parameters 

1492 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1493 

1494 parSchema = afwTable.Schema() 

1495 

1496 comma = ',' 

1497 lutFilterNameString = comma.join([n.decode('utf-8') 

1498 for n in parInfo['LUTFILTERNAMES'][0]]) 

1499 fitBandString = comma.join([n.decode('utf-8') 

1500 for n in parInfo['FITBANDS'][0]]) 

1501 

1502 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1503 lutFilterNameString, fitBandString) 

1504 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1505 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1506 lutFilterNameString, fitBandString) 

1507 

1508 fgcmDatasetDict['fgcmFitParameters'] = parCat 

1509 

1510 # Save the indices of the flagged stars 

1511 # (stars that have been (a) reserved from the fit for testing and 

1512 # (b) bad stars that have failed quality checks.) 

1513 flagStarSchema = self._makeFlagStarSchema() 

1514 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1515 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1516 

1517 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat 

1518 

1519 # Save the zeropoint information and atmospheres only if desired 

1520 if self.outputZeropoints: 

1521 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1522 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1523 

1524 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1525 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1526 

1527 fgcmDatasetDict['fgcmZeropoints'] = zptCat 

1528 

1529 # Save atmosphere values 

1530 # These are generated by the same code that generates zeropoints 

1531 atmSchema = makeAtmSchema() 

1532 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1533 

1534 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat 

1535 

1536 # Save the standard stars (if configured) 

1537 if self.outputStandards: 

1538 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1539 stdSchema = makeStdSchema(len(goodBands)) 

1540 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1541 

1542 fgcmDatasetDict['fgcmStandardStars'] = stdCat 

1543 

1544 return fgcmDatasetDict 

1545 

1546 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1547 lutFilterNameString, fitBandString): 

1548 """ 

1549 Make the parameter persistence schema 

1550 

1551 Parameters 

1552 ---------- 

1553 parInfo: `numpy.ndarray` 

1554 Parameter information returned by fgcm 

1555 pars: `numpy.ndarray` 

1556 Parameter values returned by fgcm 

1557 parSuperStarFlat: `numpy.array` 

1558 Superstar flat values returned by fgcm 

1559 lutFilterNameString: `str` 

1560 Combined string of all the lutFilterNames 

1561 fitBandString: `str` 

1562 Combined string of all the fitBands 

1563 

1564 Returns 

1565 ------- 

1566 parSchema: `afwTable.schema` 

1567 """ 

1568 

1569 parSchema = afwTable.Schema() 

1570 

1571 # parameter info section 

1572 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1573 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1574 size=len(lutFilterNameString)) 

1575 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1576 size=len(fitBandString)) 

1577 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1578 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1579 doc='Step units for ln(AOD) slope') 

1580 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1581 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1582 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1583 doc='Step units for ln(pwv) slope') 

1584 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1585 doc='Step units for ln(pwv) quadratic term') 

1586 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1587 doc='Step units for global ln(pwv) parameters') 

1588 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1589 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1590 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1591 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1592 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1593 

1594 # parameter section 

1595 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1596 size=pars['PARALPHA'].size) 

1597 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1598 size=pars['PARO3'].size) 

1599 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1600 doc='ln(Tau) intercept parameter vector', 

1601 size=pars['PARLNTAUINTERCEPT'].size) 

1602 parSchema.addField('parLnTauSlope', type='ArrayD', 

1603 doc='ln(Tau) slope parameter vector', 

1604 size=pars['PARLNTAUSLOPE'].size) 

1605 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1606 size=pars['PARLNPWVINTERCEPT'].size) 

1607 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1608 size=pars['PARLNPWVSLOPE'].size) 

1609 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1610 size=pars['PARLNPWVQUADRATIC'].size) 

1611 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1612 size=pars['PARQESYSINTERCEPT'].size) 

1613 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1614 size=pars[0]['COMPQESYSSLOPE'].size) 

1615 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1616 size=pars['PARFILTEROFFSET'].size) 

1617 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1618 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1619 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1620 doc='Global scale for retrieved ln(pwv)') 

1621 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1622 doc='Global offset for retrieved ln(pwv)') 

1623 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1624 doc='Nightly offset for retrieved ln(pwv)', 

1625 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1626 parSchema.addField('compAbsThroughput', type='ArrayD', 

1627 doc='Absolute throughput (relative to transmission curves)', 

1628 size=pars['COMPABSTHROUGHPUT'].size) 

1629 parSchema.addField('compRefOffset', type='ArrayD', 

1630 doc='Offset between reference stars and calibrated stars', 

1631 size=pars['COMPREFOFFSET'].size) 

1632 parSchema.addField('compRefSigma', type='ArrayD', 

1633 doc='Width of reference star/calibrated star distribution', 

1634 size=pars['COMPREFSIGMA'].size) 

1635 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1636 doc='Computed mirror chromaticity terms', 

1637 size=pars['COMPMIRRORCHROMATICITY'].size) 

1638 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1639 doc='Mirror chromaticity pivot mjd', 

1640 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1641 parSchema.addField('compMedianSedSlope', type='ArrayD', 

1642 doc='Computed median SED slope (per band)', 

1643 size=pars['COMPMEDIANSEDSLOPE'].size) 

1644 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1645 size=pars['COMPAPERCORRPIVOT'].size) 

1646 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1647 size=pars['COMPAPERCORRSLOPE'].size) 

1648 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1649 size=pars['COMPAPERCORRSLOPEERR'].size) 

1650 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1651 size=pars['COMPAPERCORRRANGE'].size) 

1652 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1653 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1654 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1655 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1656 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1657 size=pars['COMPMODELERRSKYPIVOT'].size) 

1658 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1659 size=pars['COMPMODELERRPARS'].size) 

1660 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1661 size=pars['COMPEXPGRAY'].size) 

1662 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1663 size=pars['COMPVARGRAY'].size) 

1664 parSchema.addField('compExpDeltaMagBkg', type='ArrayD', 

1665 doc='Computed exposure offset due to background', 

1666 size=pars['COMPEXPDELTAMAGBKG'].size) 

1667 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1668 doc='Computed number of good stars per exposure', 

1669 size=pars['COMPNGOODSTARPEREXP'].size) 

1670 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1671 size=pars['COMPSIGFGCM'].size) 

1672 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1673 size=pars['COMPSIGMACAL'].size) 

1674 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1675 size=pars['COMPRETRIEVEDLNPWV'].size) 

1676 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1677 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1678 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1679 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1680 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1681 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1682 # superstarflat section 

1683 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1684 size=4) 

1685 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1686 size=parSuperStarFlat.size) 

1687 

1688 return parSchema 

1689 

1690 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1691 lutFilterNameString, fitBandString): 

1692 """ 

1693 Make the FGCM parameter catalog for persistence 

1694 

1695 Parameters 

1696 ---------- 

1697 parSchema: `lsst.afw.table.Schema` 

1698 Parameter catalog schema 

1699 pars: `numpy.ndarray` 

1700 FGCM parameters to put into parCat 

1701 parSuperStarFlat: `numpy.array` 

1702 FGCM superstar flat array to put into parCat 

1703 lutFilterNameString: `str` 

1704 Combined string of all the lutFilterNames 

1705 fitBandString: `str` 

1706 Combined string of all the fitBands 

1707 

1708 Returns 

1709 ------- 

1710 parCat: `afwTable.BasicCatalog` 

1711 Atmosphere and instrumental model parameter catalog for persistence 

1712 """ 

1713 

1714 parCat = afwTable.BaseCatalog(parSchema) 

1715 parCat.reserve(1) 

1716 

1717 # The parameter catalog just has one row, with many columns for all the 

1718 # atmosphere and instrument fit parameters 

1719 rec = parCat.addNew() 

1720 

1721 # info section 

1722 rec['nCcd'] = parInfo['NCCD'] 

1723 rec['lutFilterNames'] = lutFilterNameString 

1724 rec['fitBands'] = fitBandString 

1725 # note these are not currently supported here. 

1726 rec['hasExternalPwv'] = 0 

1727 rec['hasExternalTau'] = 0 

1728 

1729 # parameter section 

1730 

1731 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1732 

1733 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1734 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1735 'parQeSysIntercept', 'compQeSysSlope', 

1736 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1737 'parFilterOffset', 'parFilterOffsetFitFlag', 

1738 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1739 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1740 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1741 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1742 'compModelErrSkyPivot', 'compModelErrPars', 

1743 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1744 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope', 

1745 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1746 'compRetrievedTauNight'] 

1747 

1748 for scalarName in scalarNames: 

1749 rec[scalarName] = pars[scalarName.upper()] 

1750 

1751 for arrName in arrNames: 

1752 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1753 

1754 # superstar section 

1755 rec['superstarSize'][:] = parSuperStarFlat.shape 

1756 rec['superstar'][:] = parSuperStarFlat.flatten() 

1757 

1758 return parCat 

1759 

1760 def _makeFlagStarSchema(self): 

1761 """ 

1762 Make the flagged-stars schema 

1763 

1764 Returns 

1765 ------- 

1766 flagStarSchema: `lsst.afw.table.Schema` 

1767 """ 

1768 

1769 flagStarSchema = afwTable.Schema() 

1770 

1771 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1772 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1773 

1774 return flagStarSchema 

1775 

1776 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1777 """ 

1778 Make the flagged star catalog for persistence 

1779 

1780 Parameters 

1781 ---------- 

1782 flagStarSchema: `lsst.afw.table.Schema` 

1783 Flagged star schema 

1784 flagStarStruct: `numpy.ndarray` 

1785 Flagged star structure from fgcm 

1786 

1787 Returns 

1788 ------- 

1789 flagStarCat: `lsst.afw.table.BaseCatalog` 

1790 Flagged star catalog for persistence 

1791 """ 

1792 

1793 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1794 flagStarCat.resize(flagStarStruct.size) 

1795 

1796 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1797 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1798 

1799 return flagStarCat