Coverage for python/lsst/fgcmcal/fgcmFitCycle.py: 21%

578 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-02 14:15 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import copy 

37 

38import numpy as np 

39 

40import lsst.pex.config as pexConfig 

41import lsst.pipe.base as pipeBase 

42from lsst.pipe.base import connectionTypes 

43import lsst.afw.table as afwTable 

44 

45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

46from .utilities import extractReferenceMags 

47from .utilities import makeZptSchema, makeZptCat 

48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

49from .sedterms import SedboundarytermDict, SedtermDict 

50from .focalPlaneProjector import FocalPlaneProjector 

51 

52import fgcm 

53 

54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask'] 

55 

56MULTIPLE_CYCLES_MAX = 10 

57 

58 

59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

60 dimensions=("instrument",), 

61 defaultTemplates={"previousCycleNumber": "-1", 

62 "cycleNumber": "0"}): 

63 camera = connectionTypes.PrerequisiteInput( 

64 doc="Camera instrument", 

65 name="camera", 

66 storageClass="Camera", 

67 dimensions=("instrument",), 

68 isCalibration=True, 

69 ) 

70 

71 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

72 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

73 "chromatic corrections."), 

74 name="fgcmLookUpTable", 

75 storageClass="Catalog", 

76 dimensions=("instrument",), 

77 deferLoad=True, 

78 ) 

79 

80 fgcmVisitCatalog = connectionTypes.Input( 

81 doc="Catalog of visit information for fgcm", 

82 name="fgcmVisitCatalog", 

83 storageClass="Catalog", 

84 dimensions=("instrument",), 

85 deferLoad=True, 

86 ) 

87 

88 fgcmStarObservationsParquet = connectionTypes.Input( 

89 doc=("Catalog of star observations for fgcm, in parquet format. " 

90 "Used if useParquetCatalogFormat is True."), 

91 name="fgcm_star_observations", 

92 storageClass="ArrowAstropy", 

93 dimensions=("instrument",), 

94 deferLoad=True, 

95 ) 

96 

97 fgcmStarIdsParquet = connectionTypes.Input( 

98 doc=("Catalog of fgcm calibration star IDs, in parquet format. " 

99 "Used if useParquetCatalogFormat is True."), 

100 name="fgcm_star_ids", 

101 storageClass="ArrowAstropy", 

102 dimensions=("instrument",), 

103 deferLoad=True, 

104 ) 

105 

106 fgcmReferenceStarsParquet = connectionTypes.Input( 

107 doc=("Catalog of fgcm-matched reference stars, in parquet format. " 

108 "Used if useParquetCatalogFormat is True."), 

109 name="fgcm_reference_stars", 

110 storageClass="ArrowAstropy", 

111 dimensions=("instrument",), 

112 deferLoad=True, 

113 ) 

114 

115 fgcmStarObservations = connectionTypes.Input( 

116 doc=("Catalog of star observations for fgcm; old format. " 

117 "Used if useParquetCatalogFormat is False."), 

118 name="fgcmStarObservations", 

119 storageClass="Catalog", 

120 dimensions=("instrument",), 

121 deferLoad=True, 

122 ) 

123 

124 fgcmStarIds = connectionTypes.Input( 

125 doc=("Catalog of fgcm calibration star IDs. " 

126 "Used if useParquetCatalogFormat is False."), 

127 name="fgcmStarIds", 

128 storageClass="Catalog", 

129 dimensions=("instrument",), 

130 deferLoad=True, 

131 ) 

132 

133 fgcmStarIndices = connectionTypes.Input( 

134 doc=("Catalog of fgcm calibration star indices; old format." 

135 "Used if useParquetCatalogFormat is False."), 

136 name="fgcmStarIndices", 

137 storageClass="Catalog", 

138 dimensions=("instrument",), 

139 deferLoad=True, 

140 ) 

141 

142 fgcmReferenceStars = connectionTypes.Input( 

143 doc=("Catalog of fgcm-matched reference stars; old format." 

144 "Used if useParquetCatalogFormat is False."), 

145 name="fgcmReferenceStars", 

146 storageClass="Catalog", 

147 dimensions=("instrument",), 

148 deferLoad=True, 

149 ) 

150 

151 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput( 

152 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle", 

153 name="fgcmFlaggedStars{previousCycleNumber}", 

154 storageClass="Catalog", 

155 dimensions=("instrument",), 

156 deferLoad=True, 

157 ) 

158 

159 fgcmFitParametersInput = connectionTypes.PrerequisiteInput( 

160 doc="Catalog of fgcm fit parameters from previous fit cycle", 

161 name="fgcmFitParameters{previousCycleNumber}", 

162 storageClass="Catalog", 

163 dimensions=("instrument",), 

164 deferLoad=True, 

165 ) 

166 

167 fgcmFitParameters = connectionTypes.Output( 

168 doc="Catalog of fgcm fit parameters from current fit cycle", 

169 name="fgcmFitParameters{cycleNumber}", 

170 storageClass="Catalog", 

171 dimensions=("instrument",), 

172 ) 

173 

174 fgcmFlaggedStars = connectionTypes.Output( 

175 doc="Catalog of flagged stars for fgcm calibration from current fit cycle", 

176 name="fgcmFlaggedStars{cycleNumber}", 

177 storageClass="Catalog", 

178 dimensions=("instrument",), 

179 ) 

180 

181 fgcmZeropoints = connectionTypes.Output( 

182 doc="Catalog of fgcm zeropoint data from current fit cycle", 

183 name="fgcmZeropoints{cycleNumber}", 

184 storageClass="Catalog", 

185 dimensions=("instrument",), 

186 ) 

187 

188 fgcmAtmosphereParameters = connectionTypes.Output( 

189 doc="Catalog of atmospheric fit parameters from current fit cycle", 

190 name="fgcmAtmosphereParameters{cycleNumber}", 

191 storageClass="Catalog", 

192 dimensions=("instrument",), 

193 ) 

194 

195 fgcmStandardStars = connectionTypes.Output( 

196 doc="Catalog of standard star magnitudes from current fit cycle", 

197 name="fgcmStandardStars{cycleNumber}", 

198 storageClass="SimpleCatalog", 

199 dimensions=("instrument",), 

200 ) 

201 

202 # Add connections for running multiple cycles 

203 # This uses vars() to alter the class __dict__ to programmatically 

204 # write many similar outputs. 

205 for cycle in range(MULTIPLE_CYCLES_MAX): 

206 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output( 

207 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}", 

208 name=f"fgcmFitParameters{cycle}", 

209 storageClass="Catalog", 

210 dimensions=("instrument",), 

211 ) 

212 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output( 

213 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}", 

214 name=f"fgcmFlaggedStars{cycle}", 

215 storageClass="Catalog", 

216 dimensions=("instrument",), 

217 ) 

218 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output( 

219 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}", 

220 name=f"fgcmZeropoints{cycle}", 

221 storageClass="Catalog", 

222 dimensions=("instrument",), 

223 ) 

224 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output( 

225 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}", 

226 name=f"fgcmAtmosphereParameters{cycle}", 

227 storageClass="Catalog", 

228 dimensions=("instrument",), 

229 ) 

230 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output( 

231 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}", 

232 name=f"fgcmStandardStars{cycle}", 

233 storageClass="SimpleCatalog", 

234 dimensions=("instrument",), 

235 ) 

236 

237 def __init__(self, *, config=None): 

238 super().__init__(config=config) 

239 

240 if not config.doReferenceCalibration: 

241 self.inputs.remove("fgcmReferenceStars") 

242 self.inputs.remove("fgcmReferenceStarsParquet") 

243 

244 if config.useParquetCatalogFormat: 

245 self.inputs.remove("fgcmStarObservations") 

246 self.inputs.remove("fgcmStarIds") 

247 self.inputs.remove("fgcmStarIndices") 

248 if config.doReferenceCalibration: 

249 self.inputs.remove("fgcmReferenceStars") 

250 else: 

251 self.inputs.remove("fgcmStarObservationsParquet") 

252 self.inputs.remove("fgcmStarIdsParquet") 

253 if config.doReferenceCalibration: 

254 self.inputs.remove("fgcmReferenceStarsParquet") 

255 

256 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

257 raise ValueError("cycleNumber must be of integer format") 

258 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

259 raise ValueError("previousCycleNumber must be of integer format") 

260 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

261 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

262 

263 if int(config.connections.cycleNumber) == 0: 

264 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput") 

265 self.prerequisiteInputs.remove("fgcmFitParametersInput") 

266 

267 if not self.config.doMultipleCycles: 

268 # Single-cycle run 

269 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle: 

270 self.outputs.remove("fgcmStandardStars") 

271 

272 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle: 

273 self.outputs.remove("fgcmZeropoints") 

274 self.outputs.remove("fgcmAtmosphereParameters") 

275 

276 # Remove all multiple cycle outputs 

277 for cycle in range(0, MULTIPLE_CYCLES_MAX): 

278 self.outputs.remove(f"fgcmFitParameters{cycle}") 

279 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

280 self.outputs.remove(f"fgcmZeropoints{cycle}") 

281 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

282 self.outputs.remove(f"fgcmStandardStars{cycle}") 

283 

284 else: 

285 # Multiple-cycle run 

286 # Remove single-cycle outputs 

287 self.outputs.remove("fgcmFitParameters") 

288 self.outputs.remove("fgcmFlaggedStars") 

289 self.outputs.remove("fgcmZeropoints") 

290 self.outputs.remove("fgcmAtmosphereParameters") 

291 self.outputs.remove("fgcmStandardStars") 

292 

293 # Remove outputs from cycles that are not used 

294 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1, 

295 MULTIPLE_CYCLES_MAX): 

296 self.outputs.remove(f"fgcmFitParameters{cycle}") 

297 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

298 self.outputs.remove(f"fgcmZeropoints{cycle}") 

299 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

300 self.outputs.remove(f"fgcmStandardStars{cycle}") 

301 

302 # Remove non-final-cycle outputs if necessary 

303 for cycle in range(self.config.multipleCyclesFinalCycleNumber): 

304 if not self.config.outputZeropointsBeforeFinalCycle: 

305 self.outputs.remove(f"fgcmZeropoints{cycle}") 

306 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

307 if not self.config.outputStandardsBeforeFinalCycle: 

308 self.outputs.remove(f"fgcmStandardStars{cycle}") 

309 

310 

311class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

312 pipelineConnections=FgcmFitCycleConnections): 

313 """Config for FgcmFitCycle""" 

314 

315 doMultipleCycles = pexConfig.Field( 

316 doc="Run multiple fit cycles in one task", 

317 dtype=bool, 

318 default=False, 

319 ) 

320 useParquetCatalogFormat = pexConfig.Field( 

321 doc="Use parquet catalog format?", 

322 dtype=bool, 

323 default=True, 

324 ) 

325 multipleCyclesFinalCycleNumber = pexConfig.RangeField( 

326 doc=("Final cycle number in multiple cycle mode. The initial cycle " 

327 "is 0, with limited parameters fit. The next cycle is 1 with " 

328 "full parameter fit. The final cycle is a clean-up with no " 

329 "parameters fit. There will be a total of " 

330 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final " 

331 "cycle number cannot be less than 2."), 

332 dtype=int, 

333 default=5, 

334 min=2, 

335 max=MULTIPLE_CYCLES_MAX, 

336 inclusiveMax=True, 

337 ) 

338 bands = pexConfig.ListField( 

339 doc="Bands to run calibration", 

340 dtype=str, 

341 default=[], 

342 ) 

343 fitBands = pexConfig.ListField( 

344 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

345 "the atmosphere constrained from the 'fitBands' on the same night. " 

346 "Must be a subset of `config.bands`"), 

347 dtype=str, 

348 default=[], 

349 ) 

350 requiredBands = pexConfig.ListField( 

351 doc=("Bands that are required for a star to be considered a calibration star. " 

352 "Must be a subset of `config.bands`"), 

353 dtype=str, 

354 default=[], 

355 ) 

356 physicalFilterMap = pexConfig.DictField( 

357 doc="Mapping from 'physicalFilter' to band.", 

358 keytype=str, 

359 itemtype=str, 

360 default={}, 

361 ) 

362 doReferenceCalibration = pexConfig.Field( 

363 doc="Use reference catalog as additional constraint on calibration", 

364 dtype=bool, 

365 default=True, 

366 ) 

367 refStarSnMin = pexConfig.Field( 

368 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

369 dtype=float, 

370 default=50.0, 

371 ) 

372 refStarOutlierNSig = pexConfig.Field( 

373 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

374 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

375 dtype=float, 

376 default=4.0, 

377 ) 

378 applyRefStarColorCuts = pexConfig.Field( 

379 doc=("Apply color cuts defined in ``starColorCuts`` to reference stars? " 

380 "These cuts are in addition to any cuts defined in ``refStarColorCuts``"), 

381 dtype=bool, 

382 default=True, 

383 ) 

384 useExposureReferenceOffset = pexConfig.Field( 

385 doc=("Use per-exposure (visit) offsets between calibrated stars and reference stars " 

386 "for final zeropoints? This may help uniformity for disjoint surveys."), 

387 dtype=bool, 

388 default=False, 

389 ) 

390 nCore = pexConfig.Field( 

391 doc="Number of cores to use", 

392 dtype=int, 

393 default=4, 

394 deprecated="Number of cores is deprecated as a config, and will be removed after v27. " 

395 "Please use ``pipetask run --cores-per-quantum`` instead.", 

396 ) 

397 nStarPerRun = pexConfig.Field( 

398 doc="Number of stars to run in each chunk", 

399 dtype=int, 

400 default=200000, 

401 ) 

402 nExpPerRun = pexConfig.Field( 

403 doc="Number of exposures to run in each chunk", 

404 dtype=int, 

405 default=1000, 

406 ) 

407 reserveFraction = pexConfig.Field( 

408 doc="Fraction of stars to reserve for testing", 

409 dtype=float, 

410 default=0.1, 

411 ) 

412 freezeStdAtmosphere = pexConfig.Field( 

413 doc="Freeze atmosphere parameters to standard (for testing)", 

414 dtype=bool, 

415 default=False, 

416 ) 

417 precomputeSuperStarInitialCycle = pexConfig.Field( 

418 doc="Precompute superstar flat for initial cycle", 

419 dtype=bool, 

420 default=False, 

421 ) 

422 superStarSubCcdDict = pexConfig.DictField( 

423 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

424 "Must have one entry per band."), 

425 keytype=str, 

426 itemtype=bool, 

427 default={}, 

428 ) 

429 superStarSubCcdChebyshevOrder = pexConfig.Field( 

430 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

431 "Global default is first-order polynomials, and should be overridden " 

432 "on a camera-by-camera basis depending on the ISR."), 

433 dtype=int, 

434 default=1, 

435 ) 

436 superStarSubCcdTriangular = pexConfig.Field( 

437 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

438 "suppress high-order cross terms?"), 

439 dtype=bool, 

440 default=False, 

441 ) 

442 superStarSigmaClip = pexConfig.Field( 

443 doc="Number of sigma to clip outliers when selecting for superstar flats", 

444 dtype=float, 

445 default=5.0, 

446 ) 

447 superStarPlotCcdResiduals = pexConfig.Field( 

448 doc="If plotting is enabled, should per-detector residuals be plotted? " 

449 "This may produce a lot of output, and should be used only for " 

450 "debugging purposes.", 

451 dtype=bool, 

452 default=False, 

453 ) 

454 focalPlaneSigmaClip = pexConfig.Field( 

455 doc="Number of sigma to clip outliers per focal-plane.", 

456 dtype=float, 

457 default=4.0, 

458 ) 

459 ccdGraySubCcdDict = pexConfig.DictField( 

460 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

461 "('ccd gray') on a sub-ccd scale."), 

462 keytype=str, 

463 itemtype=bool, 

464 default={}, 

465 ) 

466 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

467 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

468 dtype=int, 

469 default=1, 

470 ) 

471 ccdGraySubCcdTriangular = pexConfig.Field( 

472 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

473 "suppress high-order cross terms?"), 

474 dtype=bool, 

475 default=True, 

476 ) 

477 ccdGrayFocalPlaneDict = pexConfig.DictField( 

478 doc=("Per-band specification on whether to compute focal-plane residual " 

479 "('ccd gray') corrections."), 

480 keytype=str, 

481 itemtype=bool, 

482 default={}, 

483 ) 

484 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

485 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

486 "gray corrections. If there are fewer good CCDs then the gray " 

487 "correction is computed per-ccd."), 

488 dtype=int, 

489 default=1, 

490 ) 

491 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

492 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

493 dtype=int, 

494 default=3, 

495 ) 

496 cycleNumber = pexConfig.Field( 

497 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

498 "and stage of outlier rejection. See cookbook for details."), 

499 dtype=int, 

500 default=None, 

501 ) 

502 isFinalCycle = pexConfig.Field( 

503 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

504 "selection of stars and photometric exposures, and will output zeropoints " 

505 "and standard stars for use in fgcmOutputProducts"), 

506 dtype=bool, 

507 default=False, 

508 ) 

509 maxIterBeforeFinalCycle = pexConfig.Field( 

510 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

511 "will always be 0 in the final cycle for cleanup and final selection."), 

512 dtype=int, 

513 default=50, 

514 ) 

515 deltaMagBkgOffsetPercentile = pexConfig.Field( 

516 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

517 "offset from local background subtraction."), 

518 dtype=float, 

519 default=0.25, 

520 ) 

521 deltaMagBkgPerCcd = pexConfig.Field( 

522 doc=("Compute net offset from local background subtraction per-ccd? " 

523 "Otherwise, use computation per visit."), 

524 dtype=bool, 

525 default=False, 

526 ) 

527 utBoundary = pexConfig.Field( 

528 doc="Boundary (in UTC) from day-to-day", 

529 dtype=float, 

530 default=None, 

531 ) 

532 washMjds = pexConfig.ListField( 

533 doc="Mirror wash MJDs", 

534 dtype=float, 

535 default=(0.0,), 

536 ) 

537 epochMjds = pexConfig.ListField( 

538 doc="Epoch boundaries in MJD", 

539 dtype=float, 

540 default=(0.0,), 

541 ) 

542 minObsPerBand = pexConfig.Field( 

543 doc="Minimum good observations per band", 

544 dtype=int, 

545 default=2, 

546 ) 

547 # TODO: When DM-16511 is done, it will be possible to get the 

548 # telescope latitude directly from the camera. 

549 latitude = pexConfig.Field( 

550 doc="Observatory latitude", 

551 dtype=float, 

552 default=None, 

553 ) 

554 mirrorArea = pexConfig.Field( 

555 doc="Mirror area (square meters) of telescope. If not set, will " 

556 "try to estimate from camera.telescopeDiameter.", 

557 dtype=float, 

558 default=None, 

559 optional=True, 

560 ) 

561 defaultCameraOrientation = pexConfig.Field( 

562 doc="Default camera orientation for QA plots.", 

563 dtype=float, 

564 default=None, 

565 ) 

566 brightObsGrayMax = pexConfig.Field( 

567 doc="Maximum gray extinction to be considered bright observation", 

568 dtype=float, 

569 default=0.15, 

570 ) 

571 minStarPerCcd = pexConfig.Field( 

572 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

573 "CCDs with fewer stars will have their calibration estimated from other " 

574 "CCDs in the same visit, with zeropoint error increased accordingly."), 

575 dtype=int, 

576 default=5, 

577 ) 

578 minCcdPerExp = pexConfig.Field( 

579 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

580 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

581 dtype=int, 

582 default=5, 

583 ) 

584 maxCcdGrayErr = pexConfig.Field( 

585 doc="Maximum error on CCD gray offset to be considered photometric", 

586 dtype=float, 

587 default=0.05, 

588 ) 

589 minStarPerExp = pexConfig.Field( 

590 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

591 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

592 dtype=int, 

593 default=600, 

594 ) 

595 minExpPerNight = pexConfig.Field( 

596 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

597 dtype=int, 

598 default=10, 

599 ) 

600 expGrayInitialCut = pexConfig.Field( 

601 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

602 "observations."), 

603 dtype=float, 

604 default=-0.25, 

605 ) 

606 expGrayPhotometricCutDict = pexConfig.DictField( 

607 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

608 "('gray term') for a visit to be considered photometric. Must have one " 

609 "entry per band. Broad-band filters should be -0.05."), 

610 keytype=str, 

611 itemtype=float, 

612 default={}, 

613 ) 

614 expGrayHighCutDict = pexConfig.DictField( 

615 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

616 "('gray term') for a visit to be considered photometric. Must have one " 

617 "entry per band. Broad-band filters should be 0.2."), 

618 keytype=str, 

619 itemtype=float, 

620 default={}, 

621 ) 

622 expGrayRecoverCut = pexConfig.Field( 

623 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

624 "Visits with more gray extinction will only get CCD zeropoints if there are " 

625 "sufficient star observations (minStarPerCcd) on that CCD."), 

626 dtype=float, 

627 default=-1.0, 

628 ) 

629 expVarGrayPhotometricCutDict = pexConfig.DictField( 

630 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

631 "photometric. Must have one entry per band. Broad-band filters should be " 

632 "0.0005."), 

633 keytype=str, 

634 itemtype=float, 

635 default={}, 

636 ) 

637 expGrayErrRecoverCut = pexConfig.Field( 

638 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

639 "Visits with more gray variance will only get CCD zeropoints if there are " 

640 "sufficient star observations (minStarPerCcd) on that CCD."), 

641 dtype=float, 

642 default=0.05, 

643 ) 

644 aperCorrFitNBins = pexConfig.Field( 

645 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

646 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

647 "used if available."), 

648 dtype=int, 

649 default=10, 

650 ) 

651 aperCorrInputSlopeDict = pexConfig.DictField( 

652 doc=("Per-band specification of aperture correction input slope parameters. These " 

653 "are used on the first fit iteration, and aperture correction parameters will " 

654 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

655 "to set this when there is insufficient data to fit the parameters (e.g. " 

656 "tract mode)."), 

657 keytype=str, 

658 itemtype=float, 

659 default={}, 

660 ) 

661 sedboundaryterms = pexConfig.ConfigField( 

662 doc="Mapping from bands to SED boundary term names used is sedterms.", 

663 dtype=SedboundarytermDict, 

664 ) 

665 sedterms = pexConfig.ConfigField( 

666 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

667 dtype=SedtermDict, 

668 ) 

669 sigFgcmMaxErr = pexConfig.Field( 

670 doc="Maximum mag error for fitting sigma_FGCM", 

671 dtype=float, 

672 default=0.01, 

673 ) 

674 sigFgcmMaxEGrayDict = pexConfig.DictField( 

675 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

676 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

677 "should be 0.05."), 

678 keytype=str, 

679 itemtype=float, 

680 default={}, 

681 ) 

682 ccdGrayMaxStarErr = pexConfig.Field( 

683 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

684 "computation"), 

685 dtype=float, 

686 default=0.10, 

687 ) 

688 approxThroughputDict = pexConfig.DictField( 

689 doc=("Per-band specification of the approximate overall throughput at the start of " 

690 "calibration observations. Must have one entry per band. Typically should " 

691 "be 1.0."), 

692 keytype=str, 

693 itemtype=float, 

694 default={}, 

695 ) 

696 sigmaCalRange = pexConfig.ListField( 

697 doc="Allowed range for systematic error floor estimation", 

698 dtype=float, 

699 default=(0.001, 0.003), 

700 ) 

701 sigmaCalFitPercentile = pexConfig.ListField( 

702 doc="Magnitude percentile range to fit systematic error floor", 

703 dtype=float, 

704 default=(0.05, 0.15), 

705 ) 

706 sigmaCalPlotPercentile = pexConfig.ListField( 

707 doc="Magnitude percentile range to plot systematic error floor", 

708 dtype=float, 

709 default=(0.05, 0.95), 

710 ) 

711 sigma0Phot = pexConfig.Field( 

712 doc="Systematic error floor for all zeropoints", 

713 dtype=float, 

714 default=0.003, 

715 ) 

716 mapLongitudeRef = pexConfig.Field( 

717 doc="Reference longitude for plotting maps", 

718 dtype=float, 

719 default=0.0, 

720 ) 

721 mapNSide = pexConfig.Field( 

722 doc="Healpix nside for plotting maps", 

723 dtype=int, 

724 default=256, 

725 ) 

726 outfileBase = pexConfig.Field( 

727 doc="Filename start for plot output files", 

728 dtype=str, 

729 default=None, 

730 ) 

731 starColorCuts = pexConfig.ListField( 

732 doc=("Encoded star-color cuts (using calibration star colors). " 

733 "This is a list with each entry a string of the format " 

734 "``band1,band2,low,high`` such that only stars of color " 

735 "low < band1 - band2 < high will be used for calibration."), 

736 dtype=str, 

737 default=("NO_DATA",), 

738 ) 

739 refStarColorCuts = pexConfig.ListField( 

740 doc=("Encoded star color cuts specifically to apply to reference stars. " 

741 "This is a list with each entry a string of the format " 

742 "``band1,band2,low,high`` such that only stars of color " 

743 "low < band1 - band2 < high will be used as reference stars."), 

744 dtype=str, 

745 default=("NO_DATA",), 

746 ) 

747 colorSplitBands = pexConfig.ListField( 

748 doc="Band names to use to split stars by color. Must have 2 entries.", 

749 dtype=str, 

750 length=2, 

751 default=('g', 'i'), 

752 ) 

753 modelMagErrors = pexConfig.Field( 

754 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

755 dtype=bool, 

756 default=True, 

757 ) 

758 useQuadraticPwv = pexConfig.Field( 

759 doc="Model PWV with a quadratic term for variation through the night?", 

760 dtype=bool, 

761 default=False, 

762 ) 

763 instrumentParsPerBand = pexConfig.Field( 

764 doc=("Model instrumental parameters per band? " 

765 "Otherwise, instrumental parameters (QE changes with time) are " 

766 "shared among all bands."), 

767 dtype=bool, 

768 default=False, 

769 ) 

770 instrumentSlopeMinDeltaT = pexConfig.Field( 

771 doc=("Minimum time change (in days) between observations to use in constraining " 

772 "instrument slope."), 

773 dtype=float, 

774 default=20.0, 

775 ) 

776 fitMirrorChromaticity = pexConfig.Field( 

777 doc="Fit (intraband) mirror chromatic term?", 

778 dtype=bool, 

779 default=False, 

780 ) 

781 fitCcdChromaticityDict = pexConfig.DictField( 

782 doc="Specification on whether to compute first-order quantum efficiency (QE) " 

783 "adjustments. Key is band, and value will be True or False. Any band " 

784 "not explicitly specified will default to False.", 

785 keytype=str, 

786 itemtype=bool, 

787 default={}, 

788 ) 

789 coatingMjds = pexConfig.ListField( 

790 doc="Mirror coating dates in MJD", 

791 dtype=float, 

792 default=(0.0,), 

793 ) 

794 outputStandardsBeforeFinalCycle = pexConfig.Field( 

795 doc="Output standard stars prior to final cycle? Used in debugging.", 

796 dtype=bool, 

797 default=False, 

798 ) 

799 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

800 doc="Output standard stars prior to final cycle? Used in debugging.", 

801 dtype=bool, 

802 default=False, 

803 ) 

804 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

805 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

806 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

807 keytype=str, 

808 itemtype=bool, 

809 default={}, 

810 ) 

811 autoPhotometricCutNSig = pexConfig.Field( 

812 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

813 "Cut is based on exposure gray width (per band), unless " 

814 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

815 "repeatability is used (also per band)."), 

816 dtype=float, 

817 default=3.0, 

818 ) 

819 autoHighCutNSig = pexConfig.Field( 

820 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

821 "Cut is based on exposure gray width (per band), unless " 

822 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

823 "repeatability is used (also per band)."), 

824 dtype=float, 

825 default=4.0, 

826 ) 

827 quietMode = pexConfig.Field( 

828 doc="Be less verbose with logging.", 

829 dtype=bool, 

830 default=False, 

831 ) 

832 doPlots = pexConfig.Field( 

833 doc="Make fgcm QA plots.", 

834 dtype=bool, 

835 default=True, 

836 ) 

837 randomSeed = pexConfig.Field( 

838 doc="Random seed for fgcm for consistency in tests.", 

839 dtype=int, 

840 default=None, 

841 optional=True, 

842 ) 

843 deltaAperFitMinNgoodObs = pexConfig.Field( 

844 doc="Minimum number of good observations to use mean delta-aper values in fits.", 

845 dtype=int, 

846 default=2, 

847 ) 

848 deltaAperFitPerCcdNx = pexConfig.Field( 

849 doc=("Number of x bins per ccd when computing delta-aper background offsets. " 

850 "Only used when ``doComputeDeltaAperPerCcd`` is True."), 

851 dtype=int, 

852 default=10, 

853 ) 

854 deltaAperFitPerCcdNy = pexConfig.Field( 

855 doc=("Number of y bins per ccd when computing delta-aper background offsets. " 

856 "Only used when ``doComputeDeltaAperPerCcd`` is True."), 

857 dtype=int, 

858 default=10, 

859 ) 

860 deltaAperFitSpatialNside = pexConfig.Field( 

861 doc="Healpix nside to compute spatial delta-aper background offset maps.", 

862 dtype=int, 

863 default=64, 

864 ) 

865 deltaAperInnerRadiusArcsec = pexConfig.Field( 

866 doc=("Inner radius used to compute deltaMagAper (arcseconds). " 

867 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if " 

868 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, " 

869 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."), 

870 dtype=float, 

871 default=0.0, 

872 ) 

873 deltaAperOuterRadiusArcsec = pexConfig.Field( 

874 doc=("Outer radius used to compute deltaMagAper (arcseconds). " 

875 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if " 

876 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, " 

877 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."), 

878 dtype=float, 

879 default=0.0, 

880 ) 

881 doComputeDeltaAperPerVisit = pexConfig.Field( 

882 doc=("Do the computation of delta-aper background offsets per visit? " 

883 "Note: this option can be very slow when there are many visits."), 

884 dtype=bool, 

885 default=False, 

886 ) 

887 doComputeDeltaAperPerStar = pexConfig.Field( 

888 doc="Do the computation of delta-aper mean values per star?", 

889 dtype=bool, 

890 default=True, 

891 ) 

892 doComputeDeltaAperMap = pexConfig.Field( 

893 doc=("Do the computation of delta-aper spatial maps? " 

894 "This is only used if ``doComputeDeltaAperPerStar`` is True,"), 

895 dtype=bool, 

896 default=False, 

897 ) 

898 doComputeDeltaAperPerCcd = pexConfig.Field( 

899 doc="Do the computation of per-ccd delta-aper background offsets?", 

900 dtype=bool, 

901 default=False, 

902 ) 

903 

904 def validate(self): 

905 super().validate() 

906 

907 if self.connections.previousCycleNumber != str(self.cycleNumber - 1): 

908 msg = "cycleNumber in template must be connections.previousCycleNumber + 1" 

909 raise RuntimeError(msg) 

910 if self.connections.cycleNumber != str(self.cycleNumber): 

911 msg = "cycleNumber in template must be equal to connections.cycleNumber" 

912 raise RuntimeError(msg) 

913 

914 for band in self.fitBands: 

915 if band not in self.bands: 

916 msg = 'fitBand %s not in bands' % (band) 

917 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

918 for band in self.requiredBands: 

919 if band not in self.bands: 

920 msg = 'requiredBand %s not in bands' % (band) 

921 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

922 for band in self.colorSplitBands: 

923 if band not in self.bands: 

924 msg = 'colorSplitBand %s not in bands' % (band) 

925 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

926 for band in self.bands: 

927 if band not in self.superStarSubCcdDict: 

928 msg = 'band %s not in superStarSubCcdDict' % (band) 

929 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

930 self, msg) 

931 if band not in self.ccdGraySubCcdDict: 

932 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

933 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

934 self, msg) 

935 if band not in self.expGrayPhotometricCutDict: 

936 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

937 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

938 self, msg) 

939 if band not in self.expGrayHighCutDict: 

940 msg = 'band %s not in expGrayHighCutDict' % (band) 

941 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

942 self, msg) 

943 if band not in self.expVarGrayPhotometricCutDict: 

944 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

945 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

946 self, msg) 

947 if band not in self.sigFgcmMaxEGrayDict: 

948 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

949 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

950 self, msg) 

951 if band not in self.approxThroughputDict: 

952 msg = 'band %s not in approxThroughputDict' % (band) 

953 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

954 self, msg) 

955 if band not in self.useRepeatabilityForExpGrayCutsDict: 

956 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

957 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

958 self, msg) 

959 

960 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \ 

961 or self.doComputeDeltaAperPerCcd: 

962 if self.deltaAperInnerRadiusArcsec <= 0.0: 

963 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.' 

964 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec, 

965 self, msg) 

966 if self.deltaAperOuterRadiusArcsec <= 0.0: 

967 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.' 

968 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec, 

969 self, msg) 

970 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec: 

971 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if ' 

972 'deltaAper computations are turned on.') 

973 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec, 

974 self, msg) 

975 

976 

977class FgcmFitCycleTask(pipeBase.PipelineTask): 

978 """ 

979 Run Single fit cycle for FGCM global calibration 

980 """ 

981 

982 ConfigClass = FgcmFitCycleConfig 

983 _DefaultName = "fgcmFitCycle" 

984 

985 def __init__(self, initInputs=None, **kwargs): 

986 super().__init__(**kwargs) 

987 

988 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

989 camera = butlerQC.get(inputRefs.camera) 

990 

991 nCore = butlerQC.resources.num_cores 

992 

993 handleDict = {} 

994 

995 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

996 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

997 

998 if self.config.useParquetCatalogFormat: 

999 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservationsParquet) 

1000 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIdsParquet) 

1001 if self.config.doReferenceCalibration: 

1002 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStarsParquet) 

1003 else: 

1004 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations) 

1005 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds) 

1006 handleDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices) 

1007 if self.config.doReferenceCalibration: 

1008 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars) 

1009 if self.config.cycleNumber > 0: 

1010 handleDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput) 

1011 handleDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput) 

1012 

1013 fgcmDatasetDict = None 

1014 if self.config.doMultipleCycles: 

1015 # Run multiple cycles at once. 

1016 config = copy.copy(self.config) 

1017 config.update(cycleNumber=0) 

1018 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1): 

1019 if cycle == self.config.multipleCyclesFinalCycleNumber: 

1020 config.update(isFinalCycle=True) 

1021 

1022 if cycle > 0: 

1023 handleDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars'] 

1024 handleDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters'] 

1025 

1026 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config, nCore=nCore) 

1027 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], 

1028 getattr(outputRefs, f'fgcmFitParameters{cycle}')) 

1029 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], 

1030 getattr(outputRefs, f'fgcmFlaggedStars{cycle}')) 

1031 if self.outputZeropoints: 

1032 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], 

1033 getattr(outputRefs, f'fgcmZeropoints{cycle}')) 

1034 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 

1035 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}')) 

1036 if self.outputStandards: 

1037 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], 

1038 getattr(outputRefs, f'fgcmStandardStars{cycle}')) 

1039 else: 

1040 # Run a single cycle 

1041 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict, nCore=nCore) 

1042 

1043 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters) 

1044 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars) 

1045 if self.outputZeropoints: 

1046 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints) 

1047 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters) 

1048 if self.outputStandards: 

1049 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars) 

1050 

1051 def _fgcmFitCycle(self, camera, handleDict, config=None, nCore=1): 

1052 """ 

1053 Run the fit cycle 

1054 

1055 Parameters 

1056 ---------- 

1057 camera : `lsst.afw.cameraGeom.Camera` 

1058 handleDict : `dict` 

1059 All handles are `lsst.daf.butler.DeferredDatasetHandle` 

1060 handle dictionary with keys: 

1061 

1062 ``"fgcmLookUpTable"`` 

1063 handle for the FGCM look-up table. 

1064 ``"fgcmVisitCatalog"`` 

1065 handle for visit summary catalog. 

1066 ``"fgcmStarObservations"`` 

1067 handle for star observation catalog. 

1068 ``"fgcmStarIds"`` 

1069 handle for star id catalog. 

1070 ``"fgcmStarIndices"`` 

1071 handle for star index catalog. 

1072 ``"fgcmReferenceStars"`` 

1073 handle for matched reference star catalog. 

1074 ``"fgcmFlaggedStars"`` 

1075 handle for flagged star catalog. 

1076 ``"fgcmFitParameters"`` 

1077 handle for fit parameter catalog. 

1078 config : `lsst.pex.config.Config`, optional 

1079 Configuration to use to override self.config. 

1080 nCore : `int`, optional 

1081 Number of cores to use during fitting. 

1082 

1083 Returns 

1084 ------- 

1085 fgcmDatasetDict : `dict` 

1086 Dictionary of datasets to persist. 

1087 """ 

1088 if config is not None: 

1089 _config = config 

1090 else: 

1091 _config = self.config 

1092 

1093 # Set defaults on whether to output standards and zeropoints 

1094 self.maxIter = _config.maxIterBeforeFinalCycle 

1095 self.outputStandards = _config.outputStandardsBeforeFinalCycle 

1096 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle 

1097 self.resetFitParameters = True 

1098 

1099 if _config.isFinalCycle: 

1100 # This is the final fit cycle, so we do not want to reset fit 

1101 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

1102 # and we always want to output standards and zeropoints 

1103 self.maxIter = 0 

1104 self.outputStandards = True 

1105 self.outputZeropoints = True 

1106 self.resetFitParameters = False 

1107 

1108 lutCat = handleDict['fgcmLookUpTable'].get() 

1109 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

1110 dict(_config.physicalFilterMap)) 

1111 del lutCat 

1112 

1113 configDict = makeConfigDict(_config, self.log, camera, 

1114 self.maxIter, self.resetFitParameters, 

1115 self.outputZeropoints, 

1116 lutIndexVals[0]['FILTERNAMES'], 

1117 nCore=nCore) 

1118 

1119 # next we need the exposure/visit information 

1120 visitCat = handleDict['fgcmVisitCatalog'].get() 

1121 fgcmExpInfo = translateVisitCatalog(visitCat) 

1122 del visitCat 

1123 

1124 focalPlaneProjector = FocalPlaneProjector(camera, 

1125 self.config.defaultCameraOrientation) 

1126 

1127 noFitsDict = {'lutIndex': lutIndexVals, 

1128 'lutStd': lutStd, 

1129 'expInfo': fgcmExpInfo, 

1130 'focalPlaneProjector': focalPlaneProjector} 

1131 

1132 # set up the fitter object 

1133 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

1134 noFitsDict=noFitsDict, noOutput=True) 

1135 

1136 # create the parameter object 

1137 if (fgcmFitCycle.initialCycle): 

1138 # cycle = 0, initial cycle 

1139 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

1140 fgcmLut, 

1141 fgcmExpInfo) 

1142 else: 

1143 if isinstance(handleDict['fgcmFitParameters'], afwTable.BaseCatalog): 

1144 parCat = handleDict['fgcmFitParameters'] 

1145 else: 

1146 parCat = handleDict['fgcmFitParameters'].get() 

1147 inParInfo, inParams, inSuperStar = self._loadParameters(parCat) 

1148 del parCat 

1149 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

1150 fgcmExpInfo, 

1151 inParInfo, 

1152 inParams, 

1153 inSuperStar) 

1154 

1155 # set up the stars... 

1156 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

1157 

1158 starObs = handleDict['fgcmStarObservations'].get() 

1159 starIds = handleDict['fgcmStarIds'].get() 

1160 if not self.config.useParquetCatalogFormat: 

1161 starIndices = handleDict['fgcmStarIndices'].get() 

1162 else: 

1163 starIndices = None 

1164 

1165 # grab the flagged stars if available 

1166 if 'fgcmFlaggedStars' in handleDict: 

1167 if isinstance(handleDict['fgcmFlaggedStars'], afwTable.BaseCatalog): 

1168 flaggedStars = handleDict['fgcmFlaggedStars'] 

1169 else: 

1170 flaggedStars = handleDict['fgcmFlaggedStars'].get() 

1171 flagId = flaggedStars['objId'][:] 

1172 flagFlag = flaggedStars['objFlag'][:] 

1173 

1174 del flaggedStars 

1175 elif self.config.useParquetCatalogFormat: 

1176 # If we are using the parquet catalog format, then that means that 

1177 # reserved stars have already been flagged. We extract the flags here 

1178 # to input to fgcm, which will then be persisted (with additional 

1179 # quality flags) as the fgcmFlaggedStars datatype in subsequent 

1180 # fit cycles. 

1181 (flagged,) = (starIds['obj_flag'] > 0).nonzero() 

1182 flagId = starIds['fgcm_id'][flagged] 

1183 flagFlag = starIds['obj_flag'][flagged] 

1184 else: 

1185 flagId = None 

1186 flagFlag = None 

1187 

1188 if _config.doReferenceCalibration: 

1189 refStars = handleDict['fgcmReferenceStars'].get() 

1190 

1191 refMag, refMagErr = extractReferenceMags(refStars, 

1192 _config.bands, 

1193 _config.physicalFilterMap) 

1194 

1195 refId = refStars['fgcm_id'][:] 

1196 else: 

1197 refStars = None 

1198 refId = None 

1199 refMag = None 

1200 refMagErr = None 

1201 

1202 # match star observations to visits 

1203 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

1204 # actually be transferred into fgcm using the indexing below. 

1205 if self.config.useParquetCatalogFormat: 

1206 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit']) 

1207 else: 

1208 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

1209 

1210 # The fgcmStars.loadStars method will copy all the star information into 

1211 # special shared memory objects that will not blow up the memory usage when 

1212 # used with python multiprocessing. Once all the numbers are copied, 

1213 # it is necessary to release all references to the objects that previously 

1214 # stored the data to ensure that the garbage collector can clear the memory, 

1215 # and ensure that this memory is not copied when multiprocessing kicks in. 

1216 

1217 if self.config.useParquetCatalogFormat: 

1218 # Note that the ra/dec coordinates for the parquet format are in 

1219 # degrees, which is what fgcm expects. 

1220 fgcmStars.loadStars(fgcmPars, 

1221 starObs['visit'][:], 

1222 starObs['detector'][:], 

1223 starObs['ra'][:], 

1224 starObs['dec'][:], 

1225 starObs['inst_mag'][:], 

1226 starObs['inst_mag_err'][:], 

1227 fgcmExpInfo['FILTERNAME'][visitIndex], 

1228 starIds['fgcm_id'][:], 

1229 starIds['ra'][:], 

1230 starIds['dec'][:], 

1231 starIds['obs_arr_index'][:], 

1232 starIds['n_obs'][:], 

1233 obsX=starObs['x'][:], 

1234 obsY=starObs['y'][:], 

1235 obsDeltaMagBkg=starObs['delta_mag_bkg'][:], 

1236 obsDeltaAper=starObs['delta_mag_aper'][:], 

1237 refID=refId, 

1238 refMag=refMag, 

1239 refMagErr=refMagErr, 

1240 flagID=flagId, 

1241 flagFlag=flagFlag, 

1242 computeNobs=True) 

1243 else: 

1244 # We determine the conversion from the native units (typically radians) to 

1245 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

1246 # numpy arrays rather than Angles, which would we approximately 600x slower. 

1247 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

1248 

1249 fgcmStars.loadStars(fgcmPars, 

1250 starObs['visit'][starIndices['obsIndex']], 

1251 starObs['ccd'][starIndices['obsIndex']], 

1252 starObs['ra'][starIndices['obsIndex']] * conv, 

1253 starObs['dec'][starIndices['obsIndex']] * conv, 

1254 starObs['instMag'][starIndices['obsIndex']], 

1255 starObs['instMagErr'][starIndices['obsIndex']], 

1256 fgcmExpInfo['FILTERNAME'][visitIndex], 

1257 starIds['fgcm_id'][:], 

1258 starIds['ra'][:], 

1259 starIds['dec'][:], 

1260 starIds['obsArrIndex'][:], 

1261 starIds['nObs'][:], 

1262 obsX=starObs['x'][starIndices['obsIndex']], 

1263 obsY=starObs['y'][starIndices['obsIndex']], 

1264 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']], 

1265 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']], 

1266 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

1267 refID=refId, 

1268 refMag=refMag, 

1269 refMagErr=refMagErr, 

1270 flagID=flagId, 

1271 flagFlag=flagFlag, 

1272 computeNobs=True) 

1273 

1274 # Release all references to temporary objects holding star data (see above) 

1275 del starObs 

1276 del starIds 

1277 del starIndices 

1278 del flagId 

1279 del flagFlag 

1280 del refStars 

1281 del refId 

1282 del refMag 

1283 del refMagErr 

1284 

1285 # and set the bits in the cycle object 

1286 fgcmFitCycle.setLUT(fgcmLut) 

1287 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

1288 fgcmFitCycle.setPars(fgcmPars) 

1289 

1290 # finish the setup 

1291 fgcmFitCycle.finishSetup() 

1292 

1293 # and run 

1294 fgcmFitCycle.run() 

1295 

1296 ################## 

1297 # Persistance 

1298 ################## 

1299 

1300 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle) 

1301 

1302 # Output the config for the next cycle 

1303 # We need to make a copy since the input one has been frozen 

1304 

1305 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

1306 i, b in enumerate(_config.bands)} 

1307 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

1308 i, band in enumerate(_config.bands)} 

1309 

1310 outConfig = copy.copy(_config) 

1311 outConfig.update(cycleNumber=(_config.cycleNumber + 1), 

1312 precomputeSuperStarInitialCycle=False, 

1313 freezeStdAtmosphere=False, 

1314 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

1315 expGrayHighCutDict=updatedHighCutDict) 

1316 

1317 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber), 

1318 cycleNumber=str(_config.cycleNumber + 1)) 

1319 

1320 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

1321 outConfig.cycleNumber) 

1322 outConfig.save(configFileName) 

1323 

1324 if _config.isFinalCycle == 1: 

1325 # We are done, ready to output products 

1326 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

1327 else: 

1328 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

1329 self.log.info("Be sure to look at:") 

1330 self.log.info(" config.expGrayPhotometricCut") 

1331 self.log.info(" config.expGrayHighCut") 

1332 self.log.info("If you are satisfied with the fit, please set:") 

1333 self.log.info(" config.isFinalCycle = True") 

1334 

1335 fgcmFitCycle.freeSharedMemory() 

1336 

1337 return fgcmDatasetDict, outConfig 

1338 

1339 def _loadParameters(self, parCat): 

1340 """ 

1341 Load FGCM parameters from a previous fit cycle 

1342 

1343 Parameters 

1344 ---------- 

1345 parCat : `lsst.afw.table.BaseCatalog` 

1346 Parameter catalog in afw table form. 

1347 

1348 Returns 

1349 ------- 

1350 inParInfo: `numpy.ndarray` 

1351 Numpy array parameter information formatted for input to fgcm 

1352 inParameters: `numpy.ndarray` 

1353 Numpy array parameter values formatted for input to fgcm 

1354 inSuperStar: `numpy.array` 

1355 Superstar flat formatted for input to fgcm 

1356 """ 

1357 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1358 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1359 

1360 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1361 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1362 (parLutFilterNames.size, )), 

1363 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1364 ('LNTAUUNIT', 'f8'), 

1365 ('LNTAUSLOPEUNIT', 'f8'), 

1366 ('ALPHAUNIT', 'f8'), 

1367 ('LNPWVUNIT', 'f8'), 

1368 ('LNPWVSLOPEUNIT', 'f8'), 

1369 ('LNPWVQUADRATICUNIT', 'f8'), 

1370 ('LNPWVGLOBALUNIT', 'f8'), 

1371 ('O3UNIT', 'f8'), 

1372 ('QESYSUNIT', 'f8'), 

1373 ('FILTEROFFSETUNIT', 'f8'), 

1374 ('HASEXTERNALPWV', 'i2'), 

1375 ('HASEXTERNALTAU', 'i2')]) 

1376 inParInfo['NCCD'] = parCat['nCcd'] 

1377 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1378 inParInfo['FITBANDS'][:] = parFitBands 

1379 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1380 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1381 

1382 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1383 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1384 ('PARLNTAUINTERCEPT', 'f8', 

1385 (parCat['parLnTauIntercept'].size, )), 

1386 ('PARLNTAUSLOPE', 'f8', 

1387 (parCat['parLnTauSlope'].size, )), 

1388 ('PARLNPWVINTERCEPT', 'f8', 

1389 (parCat['parLnPwvIntercept'].size, )), 

1390 ('PARLNPWVSLOPE', 'f8', 

1391 (parCat['parLnPwvSlope'].size, )), 

1392 ('PARLNPWVQUADRATIC', 'f8', 

1393 (parCat['parLnPwvQuadratic'].size, )), 

1394 ('PARQESYSINTERCEPT', 'f8', 

1395 (parCat['parQeSysIntercept'].size, )), 

1396 ('COMPQESYSSLOPE', 'f8', 

1397 (parCat['compQeSysSlope'].size, )), 

1398 ('PARFILTEROFFSET', 'f8', 

1399 (parCat['parFilterOffset'].size, )), 

1400 ('PARFILTEROFFSETFITFLAG', 'i2', 

1401 (parCat['parFilterOffsetFitFlag'].size, )), 

1402 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1403 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1404 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1405 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1406 ('COMPABSTHROUGHPUT', 'f8', 

1407 (parCat['compAbsThroughput'].size, )), 

1408 ('COMPREFOFFSET', 'f8', 

1409 (parCat['compRefOffset'].size, )), 

1410 ('COMPREFSIGMA', 'f8', 

1411 (parCat['compRefSigma'].size, )), 

1412 ('COMPMIRRORCHROMATICITY', 'f8', 

1413 (parCat['compMirrorChromaticity'].size, )), 

1414 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1415 (parCat['mirrorChromaticityPivot'].size, )), 

1416 ('COMPCCDCHROMATICITY', 'f8', 

1417 (parCat['compCcdChromaticity'].size, )), 

1418 ('COMPMEDIANSEDSLOPE', 'f8', 

1419 (parCat['compMedianSedSlope'].size, )), 

1420 ('COMPAPERCORRPIVOT', 'f8', 

1421 (parCat['compAperCorrPivot'].size, )), 

1422 ('COMPAPERCORRSLOPE', 'f8', 

1423 (parCat['compAperCorrSlope'].size, )), 

1424 ('COMPAPERCORRSLOPEERR', 'f8', 

1425 (parCat['compAperCorrSlopeErr'].size, )), 

1426 ('COMPAPERCORRRANGE', 'f8', 

1427 (parCat['compAperCorrRange'].size, )), 

1428 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1429 (parCat['compModelErrExptimePivot'].size, )), 

1430 ('COMPMODELERRFWHMPIVOT', 'f8', 

1431 (parCat['compModelErrFwhmPivot'].size, )), 

1432 ('COMPMODELERRSKYPIVOT', 'f8', 

1433 (parCat['compModelErrSkyPivot'].size, )), 

1434 ('COMPMODELERRPARS', 'f8', 

1435 (parCat['compModelErrPars'].size, )), 

1436 ('COMPEXPGRAY', 'f8', 

1437 (parCat['compExpGray'].size, )), 

1438 ('COMPVARGRAY', 'f8', 

1439 (parCat['compVarGray'].size, )), 

1440 ('COMPEXPDELTAMAGBKG', 'f8', 

1441 (parCat['compExpDeltaMagBkg'].size, )), 

1442 ('COMPNGOODSTARPEREXP', 'i4', 

1443 (parCat['compNGoodStarPerExp'].size, )), 

1444 ('COMPEXPREFOFFSET', 'f8', 

1445 (parCat['compExpRefOffset'].size, )), 

1446 ('COMPSIGFGCM', 'f8', 

1447 (parCat['compSigFgcm'].size, )), 

1448 ('COMPSIGMACAL', 'f8', 

1449 (parCat['compSigmaCal'].size, )), 

1450 ('COMPRETRIEVEDLNPWV', 'f8', 

1451 (parCat['compRetrievedLnPwv'].size, )), 

1452 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1453 (parCat['compRetrievedLnPwvRaw'].size, )), 

1454 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1455 (parCat['compRetrievedLnPwvFlag'].size, )), 

1456 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1457 (parCat['compRetrievedTauNight'].size, )), 

1458 ('COMPEPSILON', 'f8', 

1459 (parCat['compEpsilon'].size, )), 

1460 ('COMPMEDDELTAAPER', 'f8', 

1461 (parCat['compMedDeltaAper'].size, )), 

1462 ('COMPGLOBALEPSILON', 'f4', 

1463 (parCat['compGlobalEpsilon'].size, )), 

1464 ('COMPEPSILONMAP', 'f4', 

1465 (parCat['compEpsilonMap'].size, )), 

1466 ('COMPEPSILONNSTARMAP', 'i4', 

1467 (parCat['compEpsilonNStarMap'].size, )), 

1468 ('COMPEPSILONCCDMAP', 'f4', 

1469 (parCat['compEpsilonCcdMap'].size, )), 

1470 ('COMPEPSILONCCDNSTARMAP', 'i4', 

1471 (parCat['compEpsilonCcdNStarMap'].size, ))]) 

1472 

1473 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1474 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1475 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1476 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1477 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1478 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1479 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1480 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1481 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1482 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1483 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1484 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1485 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1486 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1487 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1488 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1489 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1490 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1491 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1492 inParams['COMPCCDCHROMATICITY'][:] = parCat['compCcdChromaticity'][0, :] 

1493 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :] 

1494 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1495 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1496 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1497 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1498 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1499 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1500 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1501 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1502 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1503 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1504 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :] 

1505 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1506 inParams['COMPEXPREFOFFSET'][:] = parCat['compExpRefOffset'][0, :] 

1507 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1508 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1509 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1510 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1511 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1512 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1513 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :] 

1514 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :] 

1515 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :] 

1516 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :] 

1517 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :] 

1518 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :] 

1519 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :] 

1520 

1521 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1522 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1523 

1524 return (inParInfo, inParams, inSuperStar) 

1525 

1526 def _makeFgcmOutputDatasets(self, fgcmFitCycle): 

1527 """ 

1528 Persist FGCM datasets through the butler. 

1529 

1530 Parameters 

1531 ---------- 

1532 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1533 Fgcm Fit cycle object 

1534 """ 

1535 fgcmDatasetDict = {} 

1536 

1537 # Save the parameters 

1538 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1539 

1540 parSchema = afwTable.Schema() 

1541 

1542 comma = ',' 

1543 lutFilterNameString = comma.join([n.decode('utf-8') 

1544 for n in parInfo['LUTFILTERNAMES'][0]]) 

1545 fitBandString = comma.join([n.decode('utf-8') 

1546 for n in parInfo['FITBANDS'][0]]) 

1547 

1548 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1549 lutFilterNameString, fitBandString) 

1550 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1551 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1552 lutFilterNameString, fitBandString) 

1553 

1554 fgcmDatasetDict['fgcmFitParameters'] = parCat 

1555 

1556 # Save the indices of the flagged stars 

1557 # (stars that have been (a) reserved from the fit for testing and 

1558 # (b) bad stars that have failed quality checks.) 

1559 flagStarSchema = self._makeFlagStarSchema() 

1560 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1561 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1562 

1563 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat 

1564 

1565 # Save the zeropoint information and atmospheres only if desired 

1566 if self.outputZeropoints: 

1567 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1568 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1569 

1570 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1571 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1572 

1573 fgcmDatasetDict['fgcmZeropoints'] = zptCat 

1574 

1575 # Save atmosphere values 

1576 # These are generated by the same code that generates zeropoints 

1577 atmSchema = makeAtmSchema() 

1578 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1579 

1580 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat 

1581 

1582 # Save the standard stars (if configured) 

1583 if self.outputStandards: 

1584 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1585 stdSchema = makeStdSchema(len(goodBands)) 

1586 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1587 

1588 fgcmDatasetDict['fgcmStandardStars'] = stdCat 

1589 

1590 return fgcmDatasetDict 

1591 

1592 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1593 lutFilterNameString, fitBandString): 

1594 """ 

1595 Make the parameter persistence schema 

1596 

1597 Parameters 

1598 ---------- 

1599 parInfo: `numpy.ndarray` 

1600 Parameter information returned by fgcm 

1601 pars: `numpy.ndarray` 

1602 Parameter values returned by fgcm 

1603 parSuperStarFlat: `numpy.array` 

1604 Superstar flat values returned by fgcm 

1605 lutFilterNameString: `str` 

1606 Combined string of all the lutFilterNames 

1607 fitBandString: `str` 

1608 Combined string of all the fitBands 

1609 

1610 Returns 

1611 ------- 

1612 parSchema: `afwTable.schema` 

1613 """ 

1614 

1615 parSchema = afwTable.Schema() 

1616 

1617 # parameter info section 

1618 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1619 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1620 size=len(lutFilterNameString)) 

1621 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1622 size=len(fitBandString)) 

1623 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1624 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1625 doc='Step units for ln(AOD) slope') 

1626 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1627 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1628 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1629 doc='Step units for ln(pwv) slope') 

1630 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1631 doc='Step units for ln(pwv) quadratic term') 

1632 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1633 doc='Step units for global ln(pwv) parameters') 

1634 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1635 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1636 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1637 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1638 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1639 

1640 # parameter section 

1641 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1642 size=pars['PARALPHA'].size) 

1643 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1644 size=pars['PARO3'].size) 

1645 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1646 doc='ln(Tau) intercept parameter vector', 

1647 size=pars['PARLNTAUINTERCEPT'].size) 

1648 parSchema.addField('parLnTauSlope', type='ArrayD', 

1649 doc='ln(Tau) slope parameter vector', 

1650 size=pars['PARLNTAUSLOPE'].size) 

1651 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1652 size=pars['PARLNPWVINTERCEPT'].size) 

1653 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1654 size=pars['PARLNPWVSLOPE'].size) 

1655 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1656 size=pars['PARLNPWVQUADRATIC'].size) 

1657 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1658 size=pars['PARQESYSINTERCEPT'].size) 

1659 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1660 size=pars[0]['COMPQESYSSLOPE'].size) 

1661 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1662 size=pars['PARFILTEROFFSET'].size) 

1663 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1664 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1665 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1666 doc='Global scale for retrieved ln(pwv)') 

1667 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1668 doc='Global offset for retrieved ln(pwv)') 

1669 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1670 doc='Nightly offset for retrieved ln(pwv)', 

1671 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1672 parSchema.addField('compAbsThroughput', type='ArrayD', 

1673 doc='Absolute throughput (relative to transmission curves)', 

1674 size=pars['COMPABSTHROUGHPUT'].size) 

1675 parSchema.addField('compRefOffset', type='ArrayD', 

1676 doc='Offset between reference stars and calibrated stars', 

1677 size=pars['COMPREFOFFSET'].size) 

1678 parSchema.addField('compRefSigma', type='ArrayD', 

1679 doc='Width of reference star/calibrated star distribution', 

1680 size=pars['COMPREFSIGMA'].size) 

1681 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1682 doc='Computed mirror chromaticity terms', 

1683 size=pars['COMPMIRRORCHROMATICITY'].size) 

1684 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1685 doc='Mirror chromaticity pivot mjd', 

1686 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1687 parSchema.addField('compCcdChromaticity', type='ArrayD', 

1688 doc='Computed CCD chromaticity terms', 

1689 size=pars['COMPCCDCHROMATICITY'].size) 

1690 parSchema.addField('compMedianSedSlope', type='ArrayD', 

1691 doc='Computed median SED slope (per band)', 

1692 size=pars['COMPMEDIANSEDSLOPE'].size) 

1693 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1694 size=pars['COMPAPERCORRPIVOT'].size) 

1695 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1696 size=pars['COMPAPERCORRSLOPE'].size) 

1697 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1698 size=pars['COMPAPERCORRSLOPEERR'].size) 

1699 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1700 size=pars['COMPAPERCORRRANGE'].size) 

1701 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1702 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1703 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1704 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1705 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1706 size=pars['COMPMODELERRSKYPIVOT'].size) 

1707 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1708 size=pars['COMPMODELERRPARS'].size) 

1709 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1710 size=pars['COMPEXPGRAY'].size) 

1711 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1712 size=pars['COMPVARGRAY'].size) 

1713 parSchema.addField('compExpDeltaMagBkg', type='ArrayD', 

1714 doc='Computed exposure offset due to background', 

1715 size=pars['COMPEXPDELTAMAGBKG'].size) 

1716 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1717 doc='Computed number of good stars per exposure', 

1718 size=pars['COMPNGOODSTARPEREXP'].size) 

1719 parSchema.addField('compExpRefOffset', type='ArrayD', 

1720 doc='Computed per-visit median offset between standard stars and ref stars.', 

1721 size=pars['COMPEXPREFOFFSET'].size) 

1722 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1723 size=pars['COMPSIGFGCM'].size) 

1724 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1725 size=pars['COMPSIGMACAL'].size) 

1726 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1727 size=pars['COMPRETRIEVEDLNPWV'].size) 

1728 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1729 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1730 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1731 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1732 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1733 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1734 parSchema.addField('compEpsilon', type='ArrayD', 

1735 doc='Computed epsilon background offset per visit (nJy/arcsec2)', 

1736 size=pars['COMPEPSILON'].size) 

1737 parSchema.addField('compMedDeltaAper', type='ArrayD', 

1738 doc='Median delta mag aper per visit', 

1739 size=pars['COMPMEDDELTAAPER'].size) 

1740 parSchema.addField('compGlobalEpsilon', type='ArrayD', 

1741 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)', 

1742 size=pars['COMPGLOBALEPSILON'].size) 

1743 parSchema.addField('compEpsilonMap', type='ArrayD', 

1744 doc='Computed epsilon maps (nJy/arcsec2)', 

1745 size=pars['COMPEPSILONMAP'].size) 

1746 parSchema.addField('compEpsilonNStarMap', type='ArrayI', 

1747 doc='Number of stars per pixel in computed epsilon maps', 

1748 size=pars['COMPEPSILONNSTARMAP'].size) 

1749 parSchema.addField('compEpsilonCcdMap', type='ArrayD', 

1750 doc='Computed epsilon ccd maps (nJy/arcsec2)', 

1751 size=pars['COMPEPSILONCCDMAP'].size) 

1752 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI', 

1753 doc='Number of stars per ccd bin in epsilon ccd maps', 

1754 size=pars['COMPEPSILONCCDNSTARMAP'].size) 

1755 # superstarflat section 

1756 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1757 size=4) 

1758 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1759 size=parSuperStarFlat.size) 

1760 

1761 return parSchema 

1762 

1763 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1764 lutFilterNameString, fitBandString): 

1765 """ 

1766 Make the FGCM parameter catalog for persistence 

1767 

1768 Parameters 

1769 ---------- 

1770 parSchema: `lsst.afw.table.Schema` 

1771 Parameter catalog schema 

1772 pars: `numpy.ndarray` 

1773 FGCM parameters to put into parCat 

1774 parSuperStarFlat: `numpy.array` 

1775 FGCM superstar flat array to put into parCat 

1776 lutFilterNameString: `str` 

1777 Combined string of all the lutFilterNames 

1778 fitBandString: `str` 

1779 Combined string of all the fitBands 

1780 

1781 Returns 

1782 ------- 

1783 parCat: `afwTable.BasicCatalog` 

1784 Atmosphere and instrumental model parameter catalog for persistence 

1785 """ 

1786 

1787 parCat = afwTable.BaseCatalog(parSchema) 

1788 parCat.reserve(1) 

1789 

1790 # The parameter catalog just has one row, with many columns for all the 

1791 # atmosphere and instrument fit parameters 

1792 rec = parCat.addNew() 

1793 

1794 # info section 

1795 rec['nCcd'] = parInfo['NCCD'][0] 

1796 rec['lutFilterNames'] = lutFilterNameString 

1797 rec['fitBands'] = fitBandString 

1798 # note these are not currently supported here. 

1799 rec['hasExternalPwv'] = 0 

1800 rec['hasExternalTau'] = 0 

1801 

1802 # parameter section 

1803 

1804 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1805 

1806 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1807 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1808 'parQeSysIntercept', 'compQeSysSlope', 

1809 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1810 'parFilterOffset', 'parFilterOffsetFitFlag', 

1811 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1812 'compMirrorChromaticity', 'mirrorChromaticityPivot', 'compCcdChromaticity', 

1813 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1814 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1815 'compModelErrSkyPivot', 'compModelErrPars', 

1816 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1817 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope', 

1818 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1819 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper', 

1820 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap', 

1821 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap', 'compExpRefOffset'] 

1822 

1823 for scalarName in scalarNames: 

1824 rec[scalarName] = pars[scalarName.upper()][0] 

1825 

1826 for arrName in arrNames: 

1827 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1828 

1829 # superstar section 

1830 rec['superstarSize'][:] = parSuperStarFlat.shape 

1831 rec['superstar'][:] = parSuperStarFlat.ravel() 

1832 

1833 return parCat 

1834 

1835 def _makeFlagStarSchema(self): 

1836 """ 

1837 Make the flagged-stars schema 

1838 

1839 Returns 

1840 ------- 

1841 flagStarSchema: `lsst.afw.table.Schema` 

1842 """ 

1843 

1844 flagStarSchema = afwTable.Schema() 

1845 

1846 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1847 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1848 

1849 return flagStarSchema 

1850 

1851 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1852 """ 

1853 Make the flagged star catalog for persistence 

1854 

1855 Parameters 

1856 ---------- 

1857 flagStarSchema: `lsst.afw.table.Schema` 

1858 Flagged star schema 

1859 flagStarStruct: `numpy.ndarray` 

1860 Flagged star structure from fgcm 

1861 

1862 Returns 

1863 ------- 

1864 flagStarCat: `lsst.afw.table.BaseCatalog` 

1865 Flagged star catalog for persistence 

1866 """ 

1867 

1868 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1869 flagStarCat.resize(flagStarStruct.size) 

1870 

1871 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1872 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1873 

1874 return flagStarCat