Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import sys 

37import traceback 

38import copy 

39 

40import numpy as np 

41 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45import lsst.afw.table as afwTable 

46 

47from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

48from .utilities import extractReferenceMags 

49from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat 

50from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

51from .sedterms import SedboundarytermDict, SedtermDict 

52from .utilities import lookupStaticCalibrations 

53 

54import fgcm 

55 

56__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner'] 

57 

58MULTIPLE_CYCLES_MAX = 10 

59 

60 

61class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

62 dimensions=("instrument",), 

63 defaultTemplates={"previousCycleNumber": "-1", 

64 "cycleNumber": "0"}): 

65 camera = connectionTypes.PrerequisiteInput( 

66 doc="Camera instrument", 

67 name="camera", 

68 storageClass="Camera", 

69 dimensions=("instrument",), 

70 lookupFunction=lookupStaticCalibrations, 

71 isCalibration=True, 

72 ) 

73 

74 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

75 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

76 "chromatic corrections."), 

77 name="fgcmLookUpTable", 

78 storageClass="Catalog", 

79 dimensions=("instrument",), 

80 deferLoad=True, 

81 ) 

82 

83 fgcmVisitCatalog = connectionTypes.Input( 

84 doc="Catalog of visit information for fgcm", 

85 name="fgcmVisitCatalog", 

86 storageClass="Catalog", 

87 dimensions=("instrument",), 

88 deferLoad=True, 

89 ) 

90 

91 fgcmStarObservations = connectionTypes.Input( 

92 doc="Catalog of star observations for fgcm", 

93 name="fgcmStarObservations", 

94 storageClass="Catalog", 

95 dimensions=("instrument",), 

96 deferLoad=True, 

97 ) 

98 

99 fgcmStarIds = connectionTypes.Input( 

100 doc="Catalog of fgcm calibration star IDs", 

101 name="fgcmStarIds", 

102 storageClass="Catalog", 

103 dimensions=("instrument",), 

104 deferLoad=True, 

105 ) 

106 

107 fgcmStarIndices = connectionTypes.Input( 

108 doc="Catalog of fgcm calibration star indices", 

109 name="fgcmStarIndices", 

110 storageClass="Catalog", 

111 dimensions=("instrument",), 

112 deferLoad=True, 

113 ) 

114 

115 fgcmReferenceStars = connectionTypes.Input( 

116 doc="Catalog of fgcm-matched reference stars", 

117 name="fgcmReferenceStars", 

118 storageClass="Catalog", 

119 dimensions=("instrument",), 

120 deferLoad=True, 

121 ) 

122 

123 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput( 

124 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle", 

125 name="fgcmFlaggedStars{previousCycleNumber}", 

126 storageClass="Catalog", 

127 dimensions=("instrument",), 

128 deferLoad=True, 

129 ) 

130 

131 fgcmFitParametersInput = connectionTypes.PrerequisiteInput( 

132 doc="Catalog of fgcm fit parameters from previous fit cycle", 

133 name="fgcmFitParameters{previousCycleNumber}", 

134 storageClass="Catalog", 

135 dimensions=("instrument",), 

136 deferLoad=True, 

137 ) 

138 

139 fgcmFitParameters = connectionTypes.Output( 

140 doc="Catalog of fgcm fit parameters from current fit cycle", 

141 name="fgcmFitParameters{cycleNumber}", 

142 storageClass="Catalog", 

143 dimensions=("instrument",), 

144 ) 

145 

146 fgcmFlaggedStars = connectionTypes.Output( 

147 doc="Catalog of flagged stars for fgcm calibration from current fit cycle", 

148 name="fgcmFlaggedStars{cycleNumber}", 

149 storageClass="Catalog", 

150 dimensions=("instrument",), 

151 ) 

152 

153 fgcmZeropoints = connectionTypes.Output( 

154 doc="Catalog of fgcm zeropoint data from current fit cycle", 

155 name="fgcmZeropoints{cycleNumber}", 

156 storageClass="Catalog", 

157 dimensions=("instrument",), 

158 ) 

159 

160 fgcmAtmosphereParameters = connectionTypes.Output( 

161 doc="Catalog of atmospheric fit parameters from current fit cycle", 

162 name="fgcmAtmosphereParameters{cycleNumber}", 

163 storageClass="Catalog", 

164 dimensions=("instrument",), 

165 ) 

166 

167 fgcmStandardStars = connectionTypes.Output( 

168 doc="Catalog of standard star magnitudes from current fit cycle", 

169 name="fgcmStandardStars{cycleNumber}", 

170 storageClass="SimpleCatalog", 

171 dimensions=("instrument",), 

172 ) 

173 

174 # Add connections for running multiple cycles 

175 # This uses vars() to alter the class __dict__ to programmatically 

176 # write many similar outputs. 

177 for cycle in range(MULTIPLE_CYCLES_MAX): 

178 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output( 

179 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}", 

180 name=f"fgcmFitParameters{cycle}", 

181 storageClass="Catalog", 

182 dimensions=("instrument",), 

183 ) 

184 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output( 

185 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}", 

186 name=f"fgcmFlaggedStars{cycle}", 

187 storageClass="Catalog", 

188 dimensions=("instrument",), 

189 ) 

190 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output( 

191 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}", 

192 name=f"fgcmZeropoints{cycle}", 

193 storageClass="Catalog", 

194 dimensions=("instrument",), 

195 ) 

196 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output( 

197 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}", 

198 name=f"fgcmAtmosphereParameters{cycle}", 

199 storageClass="Catalog", 

200 dimensions=("instrument",), 

201 ) 

202 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output( 

203 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}", 

204 name=f"fgcmStandardStars{cycle}", 

205 storageClass="SimpleCatalog", 

206 dimensions=("instrument",), 

207 ) 

208 

209 def __init__(self, *, config=None): 

210 super().__init__(config=config) 

211 

212 if not config.doReferenceCalibration: 

213 self.inputs.remove("fgcmReferenceStars") 

214 

215 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

216 raise ValueError("cycleNumber must be of integer format") 

217 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

218 raise ValueError("previousCycleNumber must be of integer format") 

219 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

220 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

221 

222 if int(config.connections.cycleNumber) == 0: 

223 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput") 

224 self.prerequisiteInputs.remove("fgcmFitParametersInput") 

225 

226 if not self.config.doMultipleCycles: 

227 # Single-cycle run 

228 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle: 

229 self.outputs.remove("fgcmStandardStars") 

230 

231 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle: 

232 self.outputs.remove("fgcmZeropoints") 

233 self.outputs.remove("fgcmAtmosphereParameters") 

234 

235 # Remove all multiple cycle outputs 

236 for cycle in range(0, MULTIPLE_CYCLES_MAX): 

237 self.outputs.remove(f"fgcmFitParameters{cycle}") 

238 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

239 self.outputs.remove(f"fgcmZeropoints{cycle}") 

240 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

241 self.outputs.remove(f"fgcmStandardStars{cycle}") 

242 

243 else: 

244 # Multiple-cycle run 

245 # Remove single-cycle outputs 

246 self.outputs.remove("fgcmFitParameters") 

247 self.outputs.remove("fgcmFlaggedStars") 

248 self.outputs.remove("fgcmZeropoints") 

249 self.outputs.remove("fgcmAtmosphereParameters") 

250 self.outputs.remove("fgcmStandardStars") 

251 

252 # Remove outputs from cycles that are not used 

253 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1, 

254 MULTIPLE_CYCLES_MAX): 

255 self.outputs.remove(f"fgcmFitParameters{cycle}") 

256 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

257 self.outputs.remove(f"fgcmZeropoints{cycle}") 

258 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

259 self.outputs.remove(f"fgcmStandardStars{cycle}") 

260 

261 # Remove non-final-cycle outputs if necessary 

262 for cycle in range(self.config.multipleCyclesFinalCycleNumber): 

263 if not self.config.outputZeropointsBeforeFinalCycle: 

264 self.outputs.remove(f"fgcmZeropoints{cycle}") 

265 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

266 if not self.config.outputStandardsBeforeFinalCycle: 

267 self.outputs.remove(f"fgcmStandardStars{cycle}") 

268 

269 

270class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

271 pipelineConnections=FgcmFitCycleConnections): 

272 """Config for FgcmFitCycle""" 

273 

274 doMultipleCycles = pexConfig.Field( 

275 doc="Run multiple fit cycles in one task", 

276 dtype=bool, 

277 default=False, 

278 ) 

279 multipleCyclesFinalCycleNumber = pexConfig.RangeField( 

280 doc=("Final cycle number in multiple cycle mode. The initial cycle " 

281 "is 0, with limited parameters fit. The next cycle is 1 with " 

282 "full parameter fit. The final cycle is a clean-up with no " 

283 "parameters fit. There will be a total of " 

284 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final " 

285 "cycle number cannot be less than 2."), 

286 dtype=int, 

287 default=5, 

288 min=2, 

289 max=MULTIPLE_CYCLES_MAX, 

290 inclusiveMax=True, 

291 ) 

292 bands = pexConfig.ListField( 

293 doc="Bands to run calibration", 

294 dtype=str, 

295 default=[], 

296 ) 

297 fitFlag = pexConfig.ListField( 

298 doc=("Flag for which bands are directly constrained in the FGCM fit. " 

299 "Bands set to 0 will have the atmosphere constrained from observations " 

300 "in other bands on the same night. Must be same length as config.bands, " 

301 "and matched band-by-band."), 

302 dtype=int, 

303 default=(0,), 

304 optional=True, 

305 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

306 "It will be removed after v20. Use fitBands instead."), 

307 ) 

308 fitBands = pexConfig.ListField( 

309 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

310 "the atmosphere constrained from the 'fitBands' on the same night. " 

311 "Must be a subset of `config.bands`"), 

312 dtype=str, 

313 default=[], 

314 ) 

315 requiredFlag = pexConfig.ListField( 

316 doc=("Flag for which bands are required for a star to be considered a calibration " 

317 "star in the FGCM fit. Typically this should be the same as fitFlag. Must " 

318 "be same length as config.bands, and matched band-by-band."), 

319 dtype=int, 

320 default=(0,), 

321 optional=True, 

322 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

323 "It will be removed after v20. Use requiredBands instead."), 

324 ) 

325 requiredBands = pexConfig.ListField( 

326 doc=("Bands that are required for a star to be considered a calibration star. " 

327 "Must be a subset of `config.bands`"), 

328 dtype=str, 

329 default=[], 

330 ) 

331 filterMap = pexConfig.DictField( 

332 doc="Mapping from 'filterName' to band.", 

333 keytype=str, 

334 itemtype=str, 

335 default={}, 

336 deprecated=("This field is no longer used, and has been deprecated by " 

337 "DM-28088. It will be removed after v22. Use " 

338 "physicalFilterMap instead.") 

339 ) 

340 # The following config will not be necessary after Gen2 retirement. 

341 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which 

342 # is easiest to access in the config file. 

343 physicalFilterMap = pexConfig.DictField( 

344 doc="Mapping from 'physicalFilter' to band.", 

345 keytype=str, 

346 itemtype=str, 

347 default={}, 

348 ) 

349 doReferenceCalibration = pexConfig.Field( 

350 doc="Use reference catalog as additional constraint on calibration", 

351 dtype=bool, 

352 default=True, 

353 ) 

354 refStarSnMin = pexConfig.Field( 

355 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

356 dtype=float, 

357 default=50.0, 

358 ) 

359 refStarOutlierNSig = pexConfig.Field( 

360 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

361 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

362 dtype=float, 

363 default=4.0, 

364 ) 

365 applyRefStarColorCuts = pexConfig.Field( 

366 doc="Apply color cuts to reference stars?", 

367 dtype=bool, 

368 default=True, 

369 ) 

370 nCore = pexConfig.Field( 

371 doc="Number of cores to use", 

372 dtype=int, 

373 default=4, 

374 ) 

375 nStarPerRun = pexConfig.Field( 

376 doc="Number of stars to run in each chunk", 

377 dtype=int, 

378 default=200000, 

379 ) 

380 nExpPerRun = pexConfig.Field( 

381 doc="Number of exposures to run in each chunk", 

382 dtype=int, 

383 default=1000, 

384 ) 

385 reserveFraction = pexConfig.Field( 

386 doc="Fraction of stars to reserve for testing", 

387 dtype=float, 

388 default=0.1, 

389 ) 

390 freezeStdAtmosphere = pexConfig.Field( 

391 doc="Freeze atmosphere parameters to standard (for testing)", 

392 dtype=bool, 

393 default=False, 

394 ) 

395 precomputeSuperStarInitialCycle = pexConfig.Field( 

396 doc="Precompute superstar flat for initial cycle", 

397 dtype=bool, 

398 default=False, 

399 ) 

400 superStarSubCcd = pexConfig.Field( 

401 doc="Compute superstar flat on sub-ccd scale", 

402 dtype=bool, 

403 default=True, 

404 optional=True, 

405 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

406 "It will be removed after v20. Use superStarSubCcdDict instead."), 

407 ) 

408 superStarSubCcdDict = pexConfig.DictField( 

409 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

410 "Must have one entry per band."), 

411 keytype=str, 

412 itemtype=bool, 

413 default={}, 

414 ) 

415 superStarSubCcdChebyshevOrder = pexConfig.Field( 

416 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

417 "Global default is first-order polynomials, and should be overridden " 

418 "on a camera-by-camera basis depending on the ISR."), 

419 dtype=int, 

420 default=1, 

421 ) 

422 superStarSubCcdTriangular = pexConfig.Field( 

423 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

424 "suppress high-order cross terms?"), 

425 dtype=bool, 

426 default=False, 

427 ) 

428 superStarSigmaClip = pexConfig.Field( 

429 doc="Number of sigma to clip outliers when selecting for superstar flats", 

430 dtype=float, 

431 default=5.0, 

432 ) 

433 focalPlaneSigmaClip = pexConfig.Field( 

434 doc="Number of sigma to clip outliers per focal-plane.", 

435 dtype=float, 

436 default=4.0, 

437 ) 

438 ccdGraySubCcd = pexConfig.Field( 

439 doc="Compute CCD gray terms on sub-ccd scale", 

440 dtype=bool, 

441 default=False, 

442 optional=True, 

443 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

444 "It will be removed after v20. Use ccdGraySubCcdDict instead."), 

445 ) 

446 ccdGraySubCcdDict = pexConfig.DictField( 

447 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

448 "('ccd gray') on a sub-ccd scale."), 

449 keytype=str, 

450 itemtype=bool, 

451 default={}, 

452 ) 

453 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

454 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

455 dtype=int, 

456 default=1, 

457 ) 

458 ccdGraySubCcdTriangular = pexConfig.Field( 

459 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

460 "suppress high-order cross terms?"), 

461 dtype=bool, 

462 default=True, 

463 ) 

464 ccdGrayFocalPlaneDict = pexConfig.DictField( 

465 doc=("Per-band specification on whether to compute focal-plane residual " 

466 "('ccd gray') corrections."), 

467 keytype=str, 

468 itemtype=bool, 

469 default={}, 

470 ) 

471 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

472 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

473 "gray corrections. If there are fewer good CCDs then the gray " 

474 "correction is computed per-ccd."), 

475 dtype=int, 

476 default=1, 

477 ) 

478 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

479 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

480 dtype=int, 

481 default=3, 

482 ) 

483 cycleNumber = pexConfig.Field( 

484 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

485 "and stage of outlier rejection. See cookbook for details."), 

486 dtype=int, 

487 default=None, 

488 ) 

489 isFinalCycle = pexConfig.Field( 

490 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

491 "selection of stars and photometric exposures, and will output zeropoints " 

492 "and standard stars for use in fgcmOutputProducts"), 

493 dtype=bool, 

494 default=False, 

495 ) 

496 maxIterBeforeFinalCycle = pexConfig.Field( 

497 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

498 "will always be 0 in the final cycle for cleanup and final selection."), 

499 dtype=int, 

500 default=50, 

501 ) 

502 deltaMagBkgOffsetPercentile = pexConfig.Field( 

503 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

504 "offset from local background subtraction."), 

505 dtype=float, 

506 default=0.25, 

507 ) 

508 deltaMagBkgPerCcd = pexConfig.Field( 

509 doc=("Compute net offset from local background subtraction per-ccd? " 

510 "Otherwise, use computation per visit."), 

511 dtype=bool, 

512 default=False, 

513 ) 

514 utBoundary = pexConfig.Field( 

515 doc="Boundary (in UTC) from day-to-day", 

516 dtype=float, 

517 default=None, 

518 ) 

519 washMjds = pexConfig.ListField( 

520 doc="Mirror wash MJDs", 

521 dtype=float, 

522 default=(0.0,), 

523 ) 

524 epochMjds = pexConfig.ListField( 

525 doc="Epoch boundaries in MJD", 

526 dtype=float, 

527 default=(0.0,), 

528 ) 

529 minObsPerBand = pexConfig.Field( 

530 doc="Minimum good observations per band", 

531 dtype=int, 

532 default=2, 

533 ) 

534 # TODO: When DM-16511 is done, it will be possible to get the 

535 # telescope latitude directly from the camera. 

536 latitude = pexConfig.Field( 

537 doc="Observatory latitude", 

538 dtype=float, 

539 default=None, 

540 ) 

541 brightObsGrayMax = pexConfig.Field( 

542 doc="Maximum gray extinction to be considered bright observation", 

543 dtype=float, 

544 default=0.15, 

545 ) 

546 minStarPerCcd = pexConfig.Field( 

547 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

548 "CCDs with fewer stars will have their calibration estimated from other " 

549 "CCDs in the same visit, with zeropoint error increased accordingly."), 

550 dtype=int, 

551 default=5, 

552 ) 

553 minCcdPerExp = pexConfig.Field( 

554 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

555 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

556 dtype=int, 

557 default=5, 

558 ) 

559 maxCcdGrayErr = pexConfig.Field( 

560 doc="Maximum error on CCD gray offset to be considered photometric", 

561 dtype=float, 

562 default=0.05, 

563 ) 

564 minStarPerExp = pexConfig.Field( 

565 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

566 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

567 dtype=int, 

568 default=600, 

569 ) 

570 minExpPerNight = pexConfig.Field( 

571 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

572 dtype=int, 

573 default=10, 

574 ) 

575 expGrayInitialCut = pexConfig.Field( 

576 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

577 "observations."), 

578 dtype=float, 

579 default=-0.25, 

580 ) 

581 expGrayPhotometricCut = pexConfig.ListField( 

582 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. " 

583 "Must be same length as config.bands, and matched band-by-band."), 

584 dtype=float, 

585 default=(0.0,), 

586 optional=True, 

587 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

588 "It will be removed after v20. Use expGrayPhotometricCutDict instead."), 

589 ) 

590 expGrayPhotometricCutDict = pexConfig.DictField( 

591 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

592 "('gray term') for a visit to be considered photometric. Must have one " 

593 "entry per band. Broad-band filters should be -0.05."), 

594 keytype=str, 

595 itemtype=float, 

596 default={}, 

597 ) 

598 expGrayHighCut = pexConfig.ListField( 

599 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. " 

600 "Must be same length as config.bands, and matched band-by-band."), 

601 dtype=float, 

602 default=(0.0,), 

603 optional=True, 

604 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

605 "It will be removed after v20. Use expGrayHighCutDict instead."), 

606 ) 

607 expGrayHighCutDict = pexConfig.DictField( 

608 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

609 "('gray term') for a visit to be considered photometric. Must have one " 

610 "entry per band. Broad-band filters should be 0.2."), 

611 keytype=str, 

612 itemtype=float, 

613 default={}, 

614 ) 

615 expGrayRecoverCut = pexConfig.Field( 

616 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

617 "Visits with more gray extinction will only get CCD zeropoints if there are " 

618 "sufficient star observations (minStarPerCcd) on that CCD."), 

619 dtype=float, 

620 default=-1.0, 

621 ) 

622 expVarGrayPhotometricCut = pexConfig.Field( 

623 doc="Maximum exposure variance to be considered possibly photometric", 

624 dtype=float, 

625 default=0.0005, 

626 optional=True, 

627 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

628 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."), 

629 ) 

630 expVarGrayPhotometricCutDict = pexConfig.DictField( 

631 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

632 "photometric. Must have one entry per band. Broad-band filters should be " 

633 "0.0005."), 

634 keytype=str, 

635 itemtype=float, 

636 default={}, 

637 ) 

638 expGrayErrRecoverCut = pexConfig.Field( 

639 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

640 "Visits with more gray variance will only get CCD zeropoints if there are " 

641 "sufficient star observations (minStarPerCcd) on that CCD."), 

642 dtype=float, 

643 default=0.05, 

644 ) 

645 aperCorrFitNBins = pexConfig.Field( 

646 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

647 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

648 "used if available."), 

649 dtype=int, 

650 default=10, 

651 ) 

652 aperCorrInputSlopes = pexConfig.ListField( 

653 doc=("Aperture correction input slope parameters. These are used on the first " 

654 "fit iteration, and aperture correction parameters will be updated from " 

655 "the data if config.aperCorrFitNBins > 0. It is recommended to set this" 

656 "when there is insufficient data to fit the parameters (e.g. tract mode). " 

657 "If set, must be same length as config.bands, and matched band-by-band."), 

658 dtype=float, 

659 default=[], 

660 optional=True, 

661 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

662 "It will be removed after v20. Use aperCorrInputSlopeDict instead."), 

663 ) 

664 aperCorrInputSlopeDict = pexConfig.DictField( 

665 doc=("Per-band specification of aperture correction input slope parameters. These " 

666 "are used on the first fit iteration, and aperture correction parameters will " 

667 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

668 "to set this when there is insufficient data to fit the parameters (e.g. " 

669 "tract mode)."), 

670 keytype=str, 

671 itemtype=float, 

672 default={}, 

673 ) 

674 sedFudgeFactors = pexConfig.ListField( 

675 doc=("Fudge factors for computing linear SED from colors. Must be same length as " 

676 "config.bands, and matched band-by-band."), 

677 dtype=float, 

678 default=(0,), 

679 optional=True, 

680 deprecated=("This field has been deprecated and will be removed after v20. " 

681 "Please use sedSlopeTermMap and sedSlopeMap."), 

682 ) 

683 sedboundaryterms = pexConfig.ConfigField( 

684 doc="Mapping from bands to SED boundary term names used is sedterms.", 

685 dtype=SedboundarytermDict, 

686 ) 

687 sedterms = pexConfig.ConfigField( 

688 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

689 dtype=SedtermDict, 

690 ) 

691 sigFgcmMaxErr = pexConfig.Field( 

692 doc="Maximum mag error for fitting sigma_FGCM", 

693 dtype=float, 

694 default=0.01, 

695 ) 

696 sigFgcmMaxEGray = pexConfig.ListField( 

697 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. " 

698 "May be 1 element (same for all bands) or the same length as config.bands."), 

699 dtype=float, 

700 default=(0.05,), 

701 optional=True, 

702 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

703 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."), 

704 ) 

705 sigFgcmMaxEGrayDict = pexConfig.DictField( 

706 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

707 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

708 "should be 0.05."), 

709 keytype=str, 

710 itemtype=float, 

711 default={}, 

712 ) 

713 ccdGrayMaxStarErr = pexConfig.Field( 

714 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

715 "computation"), 

716 dtype=float, 

717 default=0.10, 

718 ) 

719 approxThroughput = pexConfig.ListField( 

720 doc=("Approximate overall throughput at start of calibration observations. " 

721 "May be 1 element (same for all bands) or the same length as config.bands."), 

722 dtype=float, 

723 default=(1.0, ), 

724 optional=True, 

725 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

726 "It will be removed after v20. Use approxThroughputDict instead."), 

727 ) 

728 approxThroughputDict = pexConfig.DictField( 

729 doc=("Per-band specification of the approximate overall throughput at the start of " 

730 "calibration observations. Must have one entry per band. Typically should " 

731 "be 1.0."), 

732 keytype=str, 

733 itemtype=float, 

734 default={}, 

735 ) 

736 sigmaCalRange = pexConfig.ListField( 

737 doc="Allowed range for systematic error floor estimation", 

738 dtype=float, 

739 default=(0.001, 0.003), 

740 ) 

741 sigmaCalFitPercentile = pexConfig.ListField( 

742 doc="Magnitude percentile range to fit systematic error floor", 

743 dtype=float, 

744 default=(0.05, 0.15), 

745 ) 

746 sigmaCalPlotPercentile = pexConfig.ListField( 

747 doc="Magnitude percentile range to plot systematic error floor", 

748 dtype=float, 

749 default=(0.05, 0.95), 

750 ) 

751 sigma0Phot = pexConfig.Field( 

752 doc="Systematic error floor for all zeropoints", 

753 dtype=float, 

754 default=0.003, 

755 ) 

756 mapLongitudeRef = pexConfig.Field( 

757 doc="Reference longitude for plotting maps", 

758 dtype=float, 

759 default=0.0, 

760 ) 

761 mapNSide = pexConfig.Field( 

762 doc="Healpix nside for plotting maps", 

763 dtype=int, 

764 default=256, 

765 ) 

766 outfileBase = pexConfig.Field( 

767 doc="Filename start for plot output files", 

768 dtype=str, 

769 default=None, 

770 ) 

771 starColorCuts = pexConfig.ListField( 

772 doc="Encoded star-color cuts (to be cleaned up)", 

773 dtype=str, 

774 default=("NO_DATA",), 

775 ) 

776 colorSplitIndices = pexConfig.ListField( 

777 doc="Band indices to use to split stars by color", 

778 dtype=int, 

779 default=None, 

780 optional=True, 

781 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

782 "It will be removed after v20. Use colorSplitBands instead."), 

783 ) 

784 colorSplitBands = pexConfig.ListField( 

785 doc="Band names to use to split stars by color. Must have 2 entries.", 

786 dtype=str, 

787 length=2, 

788 default=('g', 'i'), 

789 ) 

790 modelMagErrors = pexConfig.Field( 

791 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

792 dtype=bool, 

793 default=True, 

794 ) 

795 useQuadraticPwv = pexConfig.Field( 

796 doc="Model PWV with a quadratic term for variation through the night?", 

797 dtype=bool, 

798 default=False, 

799 ) 

800 instrumentParsPerBand = pexConfig.Field( 

801 doc=("Model instrumental parameters per band? " 

802 "Otherwise, instrumental parameters (QE changes with time) are " 

803 "shared among all bands."), 

804 dtype=bool, 

805 default=False, 

806 ) 

807 instrumentSlopeMinDeltaT = pexConfig.Field( 

808 doc=("Minimum time change (in days) between observations to use in constraining " 

809 "instrument slope."), 

810 dtype=float, 

811 default=20.0, 

812 ) 

813 fitMirrorChromaticity = pexConfig.Field( 

814 doc="Fit (intraband) mirror chromatic term?", 

815 dtype=bool, 

816 default=False, 

817 ) 

818 coatingMjds = pexConfig.ListField( 

819 doc="Mirror coating dates in MJD", 

820 dtype=float, 

821 default=(0.0,), 

822 ) 

823 outputStandardsBeforeFinalCycle = pexConfig.Field( 

824 doc="Output standard stars prior to final cycle? Used in debugging.", 

825 dtype=bool, 

826 default=False, 

827 ) 

828 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

829 doc="Output standard stars prior to final cycle? Used in debugging.", 

830 dtype=bool, 

831 default=False, 

832 ) 

833 useRepeatabilityForExpGrayCuts = pexConfig.ListField( 

834 doc=("Use star repeatability (instead of exposures) for computing photometric " 

835 "cuts? Recommended for tract mode or bands with few exposures. " 

836 "May be 1 element (same for all bands) or the same length as config.bands."), 

837 dtype=bool, 

838 default=(False,), 

839 optional=True, 

840 deprecated=("This field is no longer used, and has been deprecated by DM-23699. " 

841 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."), 

842 ) 

843 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

844 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

845 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

846 keytype=str, 

847 itemtype=bool, 

848 default={}, 

849 ) 

850 autoPhotometricCutNSig = pexConfig.Field( 

851 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

852 "Cut is based on exposure gray width (per band), unless " 

853 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

854 "repeatability is used (also per band)."), 

855 dtype=float, 

856 default=3.0, 

857 ) 

858 autoHighCutNSig = pexConfig.Field( 

859 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

860 "Cut is based on exposure gray width (per band), unless " 

861 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

862 "repeatability is used (also per band)."), 

863 dtype=float, 

864 default=4.0, 

865 ) 

866 quietMode = pexConfig.Field( 

867 doc="Be less verbose with logging.", 

868 dtype=bool, 

869 default=False, 

870 ) 

871 doPlots = pexConfig.Field( 

872 doc="Make fgcm QA plots.", 

873 dtype=bool, 

874 default=True, 

875 ) 

876 randomSeed = pexConfig.Field( 

877 doc="Random seed for fgcm for consistency in tests.", 

878 dtype=int, 

879 default=None, 

880 optional=True, 

881 ) 

882 

883 def validate(self): 

884 super().validate() 

885 

886 if self.connections.previousCycleNumber != str(self.cycleNumber - 1): 

887 msg = "cycleNumber in template must be connections.previousCycleNumber + 1" 

888 raise RuntimeError(msg) 

889 if self.connections.cycleNumber != str(self.cycleNumber): 

890 msg = "cycleNumber in template must be equal to connections.cycleNumber" 

891 raise RuntimeError(msg) 

892 

893 for band in self.fitBands: 

894 if band not in self.bands: 

895 msg = 'fitBand %s not in bands' % (band) 

896 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

897 for band in self.requiredBands: 

898 if band not in self.bands: 

899 msg = 'requiredBand %s not in bands' % (band) 

900 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

901 for band in self.colorSplitBands: 

902 if band not in self.bands: 

903 msg = 'colorSplitBand %s not in bands' % (band) 

904 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

905 for band in self.bands: 

906 if band not in self.superStarSubCcdDict: 

907 msg = 'band %s not in superStarSubCcdDict' % (band) 

908 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

909 self, msg) 

910 if band not in self.ccdGraySubCcdDict: 

911 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

912 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

913 self, msg) 

914 if band not in self.expGrayPhotometricCutDict: 

915 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

916 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

917 self, msg) 

918 if band not in self.expGrayHighCutDict: 

919 msg = 'band %s not in expGrayHighCutDict' % (band) 

920 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

921 self, msg) 

922 if band not in self.expVarGrayPhotometricCutDict: 

923 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

924 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

925 self, msg) 

926 if band not in self.sigFgcmMaxEGrayDict: 

927 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

928 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

929 self, msg) 

930 if band not in self.approxThroughputDict: 

931 msg = 'band %s not in approxThroughputDict' % (band) 

932 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

933 self, msg) 

934 if band not in self.useRepeatabilityForExpGrayCutsDict: 

935 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

936 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

937 self, msg) 

938 

939 

940class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner): 

941 """Subclass of TaskRunner for fgcmFitCycleTask 

942 

943 fgcmFitCycleTask.run() takes one argument, the butler, and uses 

944 stars and visits previously extracted from dataRefs by 

945 fgcmBuildStars. 

946 This Runner does not perform any dataRef parallelization, but the FGCM 

947 code called by the Task uses python multiprocessing (see the "ncores" 

948 config option). 

949 """ 

950 

951 @staticmethod 

952 def getTargetList(parsedCmd): 

953 """ 

954 Return a list with one element, the butler. 

955 """ 

956 return [parsedCmd.butler] 

957 

958 def __call__(self, butler): 

959 """ 

960 Parameters 

961 ---------- 

962 butler: `lsst.daf.persistence.Butler` 

963 

964 Returns 

965 ------- 

966 exitStatus: `list` with `pipeBase.Struct` 

967 exitStatus (0: success; 1: failure) 

968 """ 

969 

970 task = self.TaskClass(config=self.config, log=self.log) 

971 

972 exitStatus = 0 

973 if self.doRaise: 

974 task.runDataRef(butler) 

975 else: 

976 try: 

977 task.runDataRef(butler) 

978 except Exception as e: 

979 exitStatus = 1 

980 task.log.fatal("Failed: %s" % e) 

981 if not isinstance(e, pipeBase.TaskError): 

982 traceback.print_exc(file=sys.stderr) 

983 

984 task.writeMetadata(butler) 

985 

986 # The task does not return any results: 

987 return [pipeBase.Struct(exitStatus=exitStatus)] 

988 

989 def run(self, parsedCmd): 

990 """ 

991 Run the task, with no multiprocessing 

992 

993 Parameters 

994 ---------- 

995 parsedCmd: ArgumentParser parsed command line 

996 """ 

997 

998 resultList = [] 

999 

1000 if self.precall(parsedCmd): 

1001 targetList = self.getTargetList(parsedCmd) 

1002 # make sure that we only get 1 

1003 resultList = self(targetList[0]) 

1004 

1005 return resultList 

1006 

1007 

1008class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

1009 """ 

1010 Run Single fit cycle for FGCM global calibration 

1011 """ 

1012 

1013 ConfigClass = FgcmFitCycleConfig 

1014 RunnerClass = FgcmFitCycleRunner 

1015 _DefaultName = "fgcmFitCycle" 

1016 

1017 def __init__(self, butler=None, initInputs=None, **kwargs): 

1018 super().__init__(**kwargs) 

1019 

1020 # no saving of metadata for now 

1021 def _getMetadataName(self): 

1022 return None 

1023 

1024 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1025 camera = butlerQC.get(inputRefs.camera) 

1026 

1027 dataRefDict = {} 

1028 

1029 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

1030 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

1031 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations) 

1032 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds) 

1033 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices) 

1034 if self.config.doReferenceCalibration: 

1035 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars) 

1036 if self.config.cycleNumber > 0: 

1037 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput) 

1038 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput) 

1039 

1040 fgcmDatasetDict = None 

1041 if self.config.doMultipleCycles: 

1042 # Run multiple cycles at once. 

1043 config = copy.copy(self.config) 

1044 config.update(cycleNumber=0) 

1045 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1): 

1046 if cycle == self.config.multipleCyclesFinalCycleNumber: 

1047 config.update(isFinalCycle=True) 

1048 

1049 if cycle > 0: 

1050 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars'] 

1051 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters'] 

1052 

1053 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config) 

1054 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], 

1055 getattr(outputRefs, f'fgcmFitParameters{cycle}')) 

1056 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], 

1057 getattr(outputRefs, f'fgcmFlaggedStars{cycle}')) 

1058 if self.outputZeropoints: 

1059 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], 

1060 getattr(outputRefs, f'fgcmZeropoints{cycle}')) 

1061 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 

1062 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}')) 

1063 if self.outputStandards: 

1064 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], 

1065 getattr(outputRefs, f'fgcmStandardStars{cycle}')) 

1066 else: 

1067 # Run a single cycle 

1068 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict) 

1069 

1070 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters) 

1071 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars) 

1072 if self.outputZeropoints: 

1073 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints) 

1074 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters) 

1075 if self.outputStandards: 

1076 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars) 

1077 

1078 @pipeBase.timeMethod 

1079 def runDataRef(self, butler): 

1080 """ 

1081 Run a single fit cycle for FGCM 

1082 

1083 Parameters 

1084 ---------- 

1085 butler: `lsst.daf.persistence.Butler` 

1086 """ 

1087 self._checkDatasetsExist(butler) 

1088 

1089 dataRefDict = {} 

1090 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

1091 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

1092 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations') 

1093 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds') 

1094 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices') 

1095 if self.config.doReferenceCalibration: 

1096 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars') 

1097 if self.config.cycleNumber > 0: 

1098 lastCycle = self.config.cycleNumber - 1 

1099 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars', 

1100 fgcmcycle=lastCycle) 

1101 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters', 

1102 fgcmcycle=lastCycle) 

1103 

1104 camera = butler.get('camera') 

1105 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict) 

1106 

1107 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters', 

1108 fgcmcycle=self.config.cycleNumber) 

1109 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars', 

1110 fgcmcycle=self.config.cycleNumber) 

1111 if self.outputZeropoints: 

1112 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints', 

1113 fgcmcycle=self.config.cycleNumber) 

1114 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters', 

1115 fgcmcycle=self.config.cycleNumber) 

1116 if self.outputStandards: 

1117 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars', 

1118 fgcmcycle=self.config.cycleNumber) 

1119 

1120 def writeConfig(self, butler, clobber=False, doBackup=True): 

1121 """Write the configuration used for processing the data, or check that an existing 

1122 one is equal to the new one if present. This is an override of the regular 

1123 version from pipe_base that knows about fgcmcycle. 

1124 

1125 Parameters 

1126 ---------- 

1127 butler : `lsst.daf.persistence.Butler` 

1128 Data butler used to write the config. The config is written to dataset type 

1129 `CmdLineTask._getConfigName`. 

1130 clobber : `bool`, optional 

1131 A boolean flag that controls what happens if a config already has been saved: 

1132 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 

1133 - `False`: raise `TaskError` if this config does not match the existing config. 

1134 doBackup : `bool`, optional 

1135 Set to `True` to backup the config files if clobbering. 

1136 """ 

1137 configName = self._getConfigName() 

1138 if configName is None: 

1139 return 

1140 if clobber: 

1141 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber) 

1142 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber): 

1143 # this may be subject to a race condition; see #2789 

1144 try: 

1145 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber) 

1146 except Exception as exc: 

1147 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" % 

1148 (configName, exc)) 

1149 

1150 def logConfigMismatch(msg): 

1151 self.log.fatal("Comparing configuration: %s", msg) 

1152 

1153 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

1154 raise pipeBase.TaskError( 

1155 f"Config does not match existing task config {configName!r} on disk; tasks configurations" 

1156 " must be consistent within the same output repo (override with --clobber-config)") 

1157 else: 

1158 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber) 

1159 

1160 def _fgcmFitCycle(self, camera, dataRefDict, config=None): 

1161 """ 

1162 Run the fit cycle 

1163 

1164 Parameters 

1165 ---------- 

1166 camera : `lsst.afw.cameraGeom.Camera` 

1167 dataRefDict : `dict` 

1168 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1169 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1170 dataRef dictionary with keys: 

1171 

1172 ``"fgcmLookUpTable"`` 

1173 dataRef for the FGCM look-up table. 

1174 ``"fgcmVisitCatalog"`` 

1175 dataRef for visit summary catalog. 

1176 ``"fgcmStarObservations"`` 

1177 dataRef for star observation catalog. 

1178 ``"fgcmStarIds"`` 

1179 dataRef for star id catalog. 

1180 ``"fgcmStarIndices"`` 

1181 dataRef for star index catalog. 

1182 ``"fgcmReferenceStars"`` 

1183 dataRef for matched reference star catalog. 

1184 ``"fgcmFlaggedStars"`` 

1185 dataRef for flagged star catalog. 

1186 ``"fgcmFitParameters"`` 

1187 dataRef for fit parameter catalog. 

1188 config : `lsst.pex.config.Config`, optional 

1189 Configuration to use to override self.config. 

1190 

1191 Returns 

1192 ------- 

1193 fgcmDatasetDict : `dict` 

1194 Dictionary of datasets to persist. 

1195 """ 

1196 if config is not None: 

1197 _config = config 

1198 else: 

1199 _config = self.config 

1200 

1201 # Set defaults on whether to output standards and zeropoints 

1202 self.maxIter = _config.maxIterBeforeFinalCycle 

1203 self.outputStandards = _config.outputStandardsBeforeFinalCycle 

1204 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle 

1205 self.resetFitParameters = True 

1206 

1207 if _config.isFinalCycle: 

1208 # This is the final fit cycle, so we do not want to reset fit 

1209 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

1210 # and we always want to output standards and zeropoints 

1211 self.maxIter = 0 

1212 self.outputStandards = True 

1213 self.outputZeropoints = True 

1214 self.resetFitParameters = False 

1215 

1216 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1217 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

1218 dict(_config.physicalFilterMap)) 

1219 del lutCat 

1220 

1221 configDict = makeConfigDict(_config, self.log, camera, 

1222 self.maxIter, self.resetFitParameters, 

1223 self.outputZeropoints, 

1224 lutIndexVals[0]['FILTERNAMES']) 

1225 

1226 # next we need the exposure/visit information 

1227 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

1228 fgcmExpInfo = translateVisitCatalog(visitCat) 

1229 del visitCat 

1230 

1231 # Use the first orientation. 

1232 # TODO: DM-21215 will generalize to arbitrary camera orientations 

1233 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

1234 

1235 noFitsDict = {'lutIndex': lutIndexVals, 

1236 'lutStd': lutStd, 

1237 'expInfo': fgcmExpInfo, 

1238 'ccdOffsets': ccdOffsets} 

1239 

1240 # set up the fitter object 

1241 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

1242 noFitsDict=noFitsDict, noOutput=True) 

1243 

1244 # create the parameter object 

1245 if (fgcmFitCycle.initialCycle): 

1246 # cycle = 0, initial cycle 

1247 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

1248 fgcmLut, 

1249 fgcmExpInfo) 

1250 else: 

1251 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog): 

1252 parCat = dataRefDict['fgcmFitParameters'] 

1253 else: 

1254 parCat = dataRefDict['fgcmFitParameters'].get() 

1255 inParInfo, inParams, inSuperStar = self._loadParameters(parCat) 

1256 del parCat 

1257 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

1258 fgcmExpInfo, 

1259 inParInfo, 

1260 inParams, 

1261 inSuperStar) 

1262 

1263 # set up the stars... 

1264 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

1265 

1266 starObs = dataRefDict['fgcmStarObservations'].get() 

1267 starIds = dataRefDict['fgcmStarIds'].get() 

1268 starIndices = dataRefDict['fgcmStarIndices'].get() 

1269 

1270 # grab the flagged stars if available 

1271 if 'fgcmFlaggedStars' in dataRefDict: 

1272 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog): 

1273 flaggedStars = dataRefDict['fgcmFlaggedStars'] 

1274 else: 

1275 flaggedStars = dataRefDict['fgcmFlaggedStars'].get() 

1276 flagId = flaggedStars['objId'][:] 

1277 flagFlag = flaggedStars['objFlag'][:] 

1278 else: 

1279 flaggedStars = None 

1280 flagId = None 

1281 flagFlag = None 

1282 

1283 if _config.doReferenceCalibration: 

1284 refStars = dataRefDict['fgcmReferenceStars'].get() 

1285 

1286 refMag, refMagErr = extractReferenceMags(refStars, 

1287 _config.bands, 

1288 _config.physicalFilterMap) 

1289 refId = refStars['fgcm_id'][:] 

1290 else: 

1291 refStars = None 

1292 refId = None 

1293 refMag = None 

1294 refMagErr = None 

1295 

1296 # match star observations to visits 

1297 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

1298 # actually be transferred into fgcm using the indexing below. 

1299 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

1300 

1301 # The fgcmStars.loadStars method will copy all the star information into 

1302 # special shared memory objects that will not blow up the memory usage when 

1303 # used with python multiprocessing. Once all the numbers are copied, 

1304 # it is necessary to release all references to the objects that previously 

1305 # stored the data to ensure that the garbage collector can clear the memory, 

1306 # and ensure that this memory is not copied when multiprocessing kicks in. 

1307 

1308 # We determine the conversion from the native units (typically radians) to 

1309 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

1310 # numpy arrays rather than Angles, which would we approximately 600x slower. 

1311 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

1312 

1313 fgcmStars.loadStars(fgcmPars, 

1314 starObs['visit'][starIndices['obsIndex']], 

1315 starObs['ccd'][starIndices['obsIndex']], 

1316 starObs['ra'][starIndices['obsIndex']] * conv, 

1317 starObs['dec'][starIndices['obsIndex']] * conv, 

1318 starObs['instMag'][starIndices['obsIndex']], 

1319 starObs['instMagErr'][starIndices['obsIndex']], 

1320 fgcmExpInfo['FILTERNAME'][visitIndex], 

1321 starIds['fgcm_id'][:], 

1322 starIds['ra'][:], 

1323 starIds['dec'][:], 

1324 starIds['obsArrIndex'][:], 

1325 starIds['nObs'][:], 

1326 obsX=starObs['x'][starIndices['obsIndex']], 

1327 obsY=starObs['y'][starIndices['obsIndex']], 

1328 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']], 

1329 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

1330 refID=refId, 

1331 refMag=refMag, 

1332 refMagErr=refMagErr, 

1333 flagID=flagId, 

1334 flagFlag=flagFlag, 

1335 computeNobs=True) 

1336 

1337 # Release all references to temporary objects holding star data (see above) 

1338 del starObs 

1339 del starIds 

1340 del starIndices 

1341 del flagId 

1342 del flagFlag 

1343 del flaggedStars 

1344 del refStars 

1345 del refId 

1346 del refMag 

1347 del refMagErr 

1348 

1349 # and set the bits in the cycle object 

1350 fgcmFitCycle.setLUT(fgcmLut) 

1351 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

1352 fgcmFitCycle.setPars(fgcmPars) 

1353 

1354 # finish the setup 

1355 fgcmFitCycle.finishSetup() 

1356 

1357 # and run 

1358 fgcmFitCycle.run() 

1359 

1360 ################## 

1361 # Persistance 

1362 ################## 

1363 

1364 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle) 

1365 

1366 # Output the config for the next cycle 

1367 # We need to make a copy since the input one has been frozen 

1368 

1369 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

1370 i, b in enumerate(_config.bands)} 

1371 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

1372 i, band in enumerate(_config.bands)} 

1373 

1374 outConfig = copy.copy(_config) 

1375 outConfig.update(cycleNumber=(_config.cycleNumber + 1), 

1376 precomputeSuperStarInitialCycle=False, 

1377 freezeStdAtmosphere=False, 

1378 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

1379 expGrayHighCutDict=updatedHighCutDict) 

1380 

1381 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber), 

1382 cycleNumber=str(_config.cycleNumber + 1)) 

1383 

1384 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

1385 outConfig.cycleNumber) 

1386 outConfig.save(configFileName) 

1387 

1388 if _config.isFinalCycle == 1: 

1389 # We are done, ready to output products 

1390 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

1391 else: 

1392 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

1393 self.log.info("Be sure to look at:") 

1394 self.log.info(" config.expGrayPhotometricCut") 

1395 self.log.info(" config.expGrayHighCut") 

1396 self.log.info("If you are satisfied with the fit, please set:") 

1397 self.log.info(" config.isFinalCycle = True") 

1398 

1399 return fgcmDatasetDict, outConfig 

1400 

1401 def _checkDatasetsExist(self, butler): 

1402 """ 

1403 Check if necessary datasets exist to run fgcmFitCycle 

1404 

1405 Parameters 

1406 ---------- 

1407 butler: `lsst.daf.persistence.Butler` 

1408 

1409 Raises 

1410 ------ 

1411 RuntimeError 

1412 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 

1413 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 

1414 If cycleNumber > 0, then also checks for fgcmFitParameters, 

1415 fgcmFlaggedStars. 

1416 """ 

1417 

1418 if not butler.datasetExists('fgcmVisitCatalog'): 

1419 raise RuntimeError("Could not find fgcmVisitCatalog in repo!") 

1420 if not butler.datasetExists('fgcmStarObservations'): 

1421 raise RuntimeError("Could not find fgcmStarObservations in repo!") 

1422 if not butler.datasetExists('fgcmStarIds'): 

1423 raise RuntimeError("Could not find fgcmStarIds in repo!") 

1424 if not butler.datasetExists('fgcmStarIndices'): 

1425 raise RuntimeError("Could not find fgcmStarIndices in repo!") 

1426 if not butler.datasetExists('fgcmLookUpTable'): 

1427 raise RuntimeError("Could not find fgcmLookUpTable in repo!") 

1428 

1429 # Need additional datasets if we are not the initial cycle 

1430 if (self.config.cycleNumber > 0): 

1431 if not butler.datasetExists('fgcmFitParameters', 

1432 fgcmcycle=self.config.cycleNumber-1): 

1433 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" % 

1434 (self.config.cycleNumber-1)) 

1435 if not butler.datasetExists('fgcmFlaggedStars', 

1436 fgcmcycle=self.config.cycleNumber-1): 

1437 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" % 

1438 (self.config.cycleNumber-1)) 

1439 

1440 # And additional dataset if we want reference calibration 

1441 if self.config.doReferenceCalibration: 

1442 if not butler.datasetExists('fgcmReferenceStars'): 

1443 raise RuntimeError("Could not find fgcmReferenceStars in repo, and " 

1444 "doReferenceCalibration is True.") 

1445 

1446 def _loadParameters(self, parCat): 

1447 """ 

1448 Load FGCM parameters from a previous fit cycle 

1449 

1450 Parameters 

1451 ---------- 

1452 parCat : `lsst.afw.table.BaseCatalog` 

1453 Parameter catalog in afw table form. 

1454 

1455 Returns 

1456 ------- 

1457 inParInfo: `numpy.ndarray` 

1458 Numpy array parameter information formatted for input to fgcm 

1459 inParameters: `numpy.ndarray` 

1460 Numpy array parameter values formatted for input to fgcm 

1461 inSuperStar: `numpy.array` 

1462 Superstar flat formatted for input to fgcm 

1463 """ 

1464 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1465 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1466 

1467 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1468 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1469 (parLutFilterNames.size, )), 

1470 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1471 ('LNTAUUNIT', 'f8'), 

1472 ('LNTAUSLOPEUNIT', 'f8'), 

1473 ('ALPHAUNIT', 'f8'), 

1474 ('LNPWVUNIT', 'f8'), 

1475 ('LNPWVSLOPEUNIT', 'f8'), 

1476 ('LNPWVQUADRATICUNIT', 'f8'), 

1477 ('LNPWVGLOBALUNIT', 'f8'), 

1478 ('O3UNIT', 'f8'), 

1479 ('QESYSUNIT', 'f8'), 

1480 ('FILTEROFFSETUNIT', 'f8'), 

1481 ('HASEXTERNALPWV', 'i2'), 

1482 ('HASEXTERNALTAU', 'i2')]) 

1483 inParInfo['NCCD'] = parCat['nCcd'] 

1484 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1485 inParInfo['FITBANDS'][:] = parFitBands 

1486 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1487 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1488 

1489 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1490 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1491 ('PARLNTAUINTERCEPT', 'f8', 

1492 (parCat['parLnTauIntercept'].size, )), 

1493 ('PARLNTAUSLOPE', 'f8', 

1494 (parCat['parLnTauSlope'].size, )), 

1495 ('PARLNPWVINTERCEPT', 'f8', 

1496 (parCat['parLnPwvIntercept'].size, )), 

1497 ('PARLNPWVSLOPE', 'f8', 

1498 (parCat['parLnPwvSlope'].size, )), 

1499 ('PARLNPWVQUADRATIC', 'f8', 

1500 (parCat['parLnPwvQuadratic'].size, )), 

1501 ('PARQESYSINTERCEPT', 'f8', 

1502 (parCat['parQeSysIntercept'].size, )), 

1503 ('COMPQESYSSLOPE', 'f8', 

1504 (parCat['compQeSysSlope'].size, )), 

1505 ('PARFILTEROFFSET', 'f8', 

1506 (parCat['parFilterOffset'].size, )), 

1507 ('PARFILTEROFFSETFITFLAG', 'i2', 

1508 (parCat['parFilterOffsetFitFlag'].size, )), 

1509 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1510 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1511 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1512 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1513 ('COMPABSTHROUGHPUT', 'f8', 

1514 (parCat['compAbsThroughput'].size, )), 

1515 ('COMPREFOFFSET', 'f8', 

1516 (parCat['compRefOffset'].size, )), 

1517 ('COMPREFSIGMA', 'f8', 

1518 (parCat['compRefSigma'].size, )), 

1519 ('COMPMIRRORCHROMATICITY', 'f8', 

1520 (parCat['compMirrorChromaticity'].size, )), 

1521 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1522 (parCat['mirrorChromaticityPivot'].size, )), 

1523 ('COMPMEDIANSEDSLOPE', 'f8', 

1524 (parCat['compMedianSedSlope'].size, )), 

1525 ('COMPAPERCORRPIVOT', 'f8', 

1526 (parCat['compAperCorrPivot'].size, )), 

1527 ('COMPAPERCORRSLOPE', 'f8', 

1528 (parCat['compAperCorrSlope'].size, )), 

1529 ('COMPAPERCORRSLOPEERR', 'f8', 

1530 (parCat['compAperCorrSlopeErr'].size, )), 

1531 ('COMPAPERCORRRANGE', 'f8', 

1532 (parCat['compAperCorrRange'].size, )), 

1533 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1534 (parCat['compModelErrExptimePivot'].size, )), 

1535 ('COMPMODELERRFWHMPIVOT', 'f8', 

1536 (parCat['compModelErrFwhmPivot'].size, )), 

1537 ('COMPMODELERRSKYPIVOT', 'f8', 

1538 (parCat['compModelErrSkyPivot'].size, )), 

1539 ('COMPMODELERRPARS', 'f8', 

1540 (parCat['compModelErrPars'].size, )), 

1541 ('COMPEXPGRAY', 'f8', 

1542 (parCat['compExpGray'].size, )), 

1543 ('COMPVARGRAY', 'f8', 

1544 (parCat['compVarGray'].size, )), 

1545 ('COMPEXPDELTAMAGBKG', 'f8', 

1546 (parCat['compExpDeltaMagBkg'].size, )), 

1547 ('COMPNGOODSTARPEREXP', 'i4', 

1548 (parCat['compNGoodStarPerExp'].size, )), 

1549 ('COMPSIGFGCM', 'f8', 

1550 (parCat['compSigFgcm'].size, )), 

1551 ('COMPSIGMACAL', 'f8', 

1552 (parCat['compSigmaCal'].size, )), 

1553 ('COMPRETRIEVEDLNPWV', 'f8', 

1554 (parCat['compRetrievedLnPwv'].size, )), 

1555 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1556 (parCat['compRetrievedLnPwvRaw'].size, )), 

1557 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1558 (parCat['compRetrievedLnPwvFlag'].size, )), 

1559 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1560 (parCat['compRetrievedTauNight'].size, ))]) 

1561 

1562 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1563 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1564 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1565 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1566 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1567 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1568 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1569 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1570 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1571 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1572 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1573 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1574 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1575 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1576 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1577 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1578 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1579 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1580 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1581 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :] 

1582 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1583 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1584 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1585 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1586 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1587 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1588 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1589 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1590 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1591 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1592 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :] 

1593 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1594 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1595 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1596 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1597 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1598 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1599 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1600 

1601 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1602 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1603 

1604 return (inParInfo, inParams, inSuperStar) 

1605 

1606 def _makeFgcmOutputDatasets(self, fgcmFitCycle): 

1607 """ 

1608 Persist FGCM datasets through the butler. 

1609 

1610 Parameters 

1611 ---------- 

1612 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1613 Fgcm Fit cycle object 

1614 """ 

1615 fgcmDatasetDict = {} 

1616 

1617 # Save the parameters 

1618 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1619 

1620 parSchema = afwTable.Schema() 

1621 

1622 comma = ',' 

1623 lutFilterNameString = comma.join([n.decode('utf-8') 

1624 for n in parInfo['LUTFILTERNAMES'][0]]) 

1625 fitBandString = comma.join([n.decode('utf-8') 

1626 for n in parInfo['FITBANDS'][0]]) 

1627 

1628 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1629 lutFilterNameString, fitBandString) 

1630 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1631 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1632 lutFilterNameString, fitBandString) 

1633 

1634 fgcmDatasetDict['fgcmFitParameters'] = parCat 

1635 

1636 # Save the indices of the flagged stars 

1637 # (stars that have been (a) reserved from the fit for testing and 

1638 # (b) bad stars that have failed quality checks.) 

1639 flagStarSchema = self._makeFlagStarSchema() 

1640 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1641 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1642 

1643 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat 

1644 

1645 # Save the zeropoint information and atmospheres only if desired 

1646 if self.outputZeropoints: 

1647 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1648 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1649 

1650 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1651 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1652 

1653 fgcmDatasetDict['fgcmZeropoints'] = zptCat 

1654 

1655 # Save atmosphere values 

1656 # These are generated by the same code that generates zeropoints 

1657 atmSchema = makeAtmSchema() 

1658 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1659 

1660 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat 

1661 

1662 # Save the standard stars (if configured) 

1663 if self.outputStandards: 

1664 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1665 stdSchema = makeStdSchema(len(goodBands)) 

1666 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1667 

1668 fgcmDatasetDict['fgcmStandardStars'] = stdCat 

1669 

1670 return fgcmDatasetDict 

1671 

1672 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1673 lutFilterNameString, fitBandString): 

1674 """ 

1675 Make the parameter persistence schema 

1676 

1677 Parameters 

1678 ---------- 

1679 parInfo: `numpy.ndarray` 

1680 Parameter information returned by fgcm 

1681 pars: `numpy.ndarray` 

1682 Parameter values returned by fgcm 

1683 parSuperStarFlat: `numpy.array` 

1684 Superstar flat values returned by fgcm 

1685 lutFilterNameString: `str` 

1686 Combined string of all the lutFilterNames 

1687 fitBandString: `str` 

1688 Combined string of all the fitBands 

1689 

1690 Returns 

1691 ------- 

1692 parSchema: `afwTable.schema` 

1693 """ 

1694 

1695 parSchema = afwTable.Schema() 

1696 

1697 # parameter info section 

1698 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1699 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1700 size=len(lutFilterNameString)) 

1701 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1702 size=len(fitBandString)) 

1703 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1704 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1705 doc='Step units for ln(AOD) slope') 

1706 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1707 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1708 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1709 doc='Step units for ln(pwv) slope') 

1710 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1711 doc='Step units for ln(pwv) quadratic term') 

1712 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1713 doc='Step units for global ln(pwv) parameters') 

1714 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1715 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1716 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1717 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1718 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1719 

1720 # parameter section 

1721 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1722 size=pars['PARALPHA'].size) 

1723 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1724 size=pars['PARO3'].size) 

1725 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1726 doc='ln(Tau) intercept parameter vector', 

1727 size=pars['PARLNTAUINTERCEPT'].size) 

1728 parSchema.addField('parLnTauSlope', type='ArrayD', 

1729 doc='ln(Tau) slope parameter vector', 

1730 size=pars['PARLNTAUSLOPE'].size) 

1731 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1732 size=pars['PARLNPWVINTERCEPT'].size) 

1733 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1734 size=pars['PARLNPWVSLOPE'].size) 

1735 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1736 size=pars['PARLNPWVQUADRATIC'].size) 

1737 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1738 size=pars['PARQESYSINTERCEPT'].size) 

1739 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1740 size=pars[0]['COMPQESYSSLOPE'].size) 

1741 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1742 size=pars['PARFILTEROFFSET'].size) 

1743 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1744 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1745 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1746 doc='Global scale for retrieved ln(pwv)') 

1747 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1748 doc='Global offset for retrieved ln(pwv)') 

1749 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1750 doc='Nightly offset for retrieved ln(pwv)', 

1751 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1752 parSchema.addField('compAbsThroughput', type='ArrayD', 

1753 doc='Absolute throughput (relative to transmission curves)', 

1754 size=pars['COMPABSTHROUGHPUT'].size) 

1755 parSchema.addField('compRefOffset', type='ArrayD', 

1756 doc='Offset between reference stars and calibrated stars', 

1757 size=pars['COMPREFOFFSET'].size) 

1758 parSchema.addField('compRefSigma', type='ArrayD', 

1759 doc='Width of reference star/calibrated star distribution', 

1760 size=pars['COMPREFSIGMA'].size) 

1761 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1762 doc='Computed mirror chromaticity terms', 

1763 size=pars['COMPMIRRORCHROMATICITY'].size) 

1764 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1765 doc='Mirror chromaticity pivot mjd', 

1766 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1767 parSchema.addField('compMedianSedSlope', type='ArrayD', 

1768 doc='Computed median SED slope (per band)', 

1769 size=pars['COMPMEDIANSEDSLOPE'].size) 

1770 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1771 size=pars['COMPAPERCORRPIVOT'].size) 

1772 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1773 size=pars['COMPAPERCORRSLOPE'].size) 

1774 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1775 size=pars['COMPAPERCORRSLOPEERR'].size) 

1776 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1777 size=pars['COMPAPERCORRRANGE'].size) 

1778 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1779 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1780 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1781 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1782 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1783 size=pars['COMPMODELERRSKYPIVOT'].size) 

1784 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1785 size=pars['COMPMODELERRPARS'].size) 

1786 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1787 size=pars['COMPEXPGRAY'].size) 

1788 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1789 size=pars['COMPVARGRAY'].size) 

1790 parSchema.addField('compExpDeltaMagBkg', type='ArrayD', 

1791 doc='Computed exposure offset due to background', 

1792 size=pars['COMPEXPDELTAMAGBKG'].size) 

1793 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1794 doc='Computed number of good stars per exposure', 

1795 size=pars['COMPNGOODSTARPEREXP'].size) 

1796 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1797 size=pars['COMPSIGFGCM'].size) 

1798 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1799 size=pars['COMPSIGMACAL'].size) 

1800 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1801 size=pars['COMPRETRIEVEDLNPWV'].size) 

1802 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1803 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1804 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1805 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1806 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1807 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1808 # superstarflat section 

1809 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1810 size=4) 

1811 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1812 size=parSuperStarFlat.size) 

1813 

1814 return parSchema 

1815 

1816 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1817 lutFilterNameString, fitBandString): 

1818 """ 

1819 Make the FGCM parameter catalog for persistence 

1820 

1821 Parameters 

1822 ---------- 

1823 parSchema: `lsst.afw.table.Schema` 

1824 Parameter catalog schema 

1825 pars: `numpy.ndarray` 

1826 FGCM parameters to put into parCat 

1827 parSuperStarFlat: `numpy.array` 

1828 FGCM superstar flat array to put into parCat 

1829 lutFilterNameString: `str` 

1830 Combined string of all the lutFilterNames 

1831 fitBandString: `str` 

1832 Combined string of all the fitBands 

1833 

1834 Returns 

1835 ------- 

1836 parCat: `afwTable.BasicCatalog` 

1837 Atmosphere and instrumental model parameter catalog for persistence 

1838 """ 

1839 

1840 parCat = afwTable.BaseCatalog(parSchema) 

1841 parCat.reserve(1) 

1842 

1843 # The parameter catalog just has one row, with many columns for all the 

1844 # atmosphere and instrument fit parameters 

1845 rec = parCat.addNew() 

1846 

1847 # info section 

1848 rec['nCcd'] = parInfo['NCCD'] 

1849 rec['lutFilterNames'] = lutFilterNameString 

1850 rec['fitBands'] = fitBandString 

1851 # note these are not currently supported here. 

1852 rec['hasExternalPwv'] = 0 

1853 rec['hasExternalTau'] = 0 

1854 

1855 # parameter section 

1856 

1857 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1858 

1859 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1860 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1861 'parQeSysIntercept', 'compQeSysSlope', 

1862 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1863 'parFilterOffset', 'parFilterOffsetFitFlag', 

1864 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1865 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1866 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1867 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1868 'compModelErrSkyPivot', 'compModelErrPars', 

1869 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1870 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope', 

1871 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1872 'compRetrievedTauNight'] 

1873 

1874 for scalarName in scalarNames: 

1875 rec[scalarName] = pars[scalarName.upper()] 

1876 

1877 for arrName in arrNames: 

1878 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1879 

1880 # superstar section 

1881 rec['superstarSize'][:] = parSuperStarFlat.shape 

1882 rec['superstar'][:] = parSuperStarFlat.flatten() 

1883 

1884 return parCat 

1885 

1886 def _makeFlagStarSchema(self): 

1887 """ 

1888 Make the flagged-stars schema 

1889 

1890 Returns 

1891 ------- 

1892 flagStarSchema: `lsst.afw.table.Schema` 

1893 """ 

1894 

1895 flagStarSchema = afwTable.Schema() 

1896 

1897 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1898 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1899 

1900 return flagStarSchema 

1901 

1902 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1903 """ 

1904 Make the flagged star catalog for persistence 

1905 

1906 Parameters 

1907 ---------- 

1908 flagStarSchema: `lsst.afw.table.Schema` 

1909 Flagged star schema 

1910 flagStarStruct: `numpy.ndarray` 

1911 Flagged star structure from fgcm 

1912 

1913 Returns 

1914 ------- 

1915 flagStarCat: `lsst.afw.table.BaseCatalog` 

1916 Flagged star catalog for persistence 

1917 """ 

1918 

1919 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1920 flagStarCat.resize(flagStarStruct.size) 

1921 

1922 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1923 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1924 

1925 return flagStarCat