Coverage for python/lsst/fgcmcal/fgcmFitCycle.py: 20%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

592 statements  

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Perform a single fit cycle of FGCM. 

24 

25This task runs a single "fit cycle" of fgcm. Prior to running this task 

26one must run both fgcmMakeLut (to construct the atmosphere and instrumental 

27look-up-table) and fgcmBuildStars (to extract visits and star observations 

28for the global fit). 

29 

30The fgcmFitCycle is meant to be run multiple times, and is tracked by the 

31'cycleNumber'. After each run of the fit cycle, diagnostic plots should 

32be inspected to set parameters for outlier rejection on the following 

33cycle. Please see the fgcmcal Cookbook for details. 

34""" 

35 

36import sys 

37import traceback 

38import copy 

39 

40import numpy as np 

41 

42import lsst.pex.config as pexConfig 

43import lsst.pipe.base as pipeBase 

44from lsst.pipe.base import connectionTypes 

45import lsst.afw.table as afwTable 

46 

47from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog 

48from .utilities import extractReferenceMags 

49from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat 

50from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat 

51from .sedterms import SedboundarytermDict, SedtermDict 

52from .utilities import lookupStaticCalibrations 

53 

54import fgcm 

55 

56__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner'] 

57 

58MULTIPLE_CYCLES_MAX = 10 

59 

60 

61class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections, 

62 dimensions=("instrument",), 

63 defaultTemplates={"previousCycleNumber": "-1", 

64 "cycleNumber": "0"}): 

65 camera = connectionTypes.PrerequisiteInput( 

66 doc="Camera instrument", 

67 name="camera", 

68 storageClass="Camera", 

69 dimensions=("instrument",), 

70 lookupFunction=lookupStaticCalibrations, 

71 isCalibration=True, 

72 ) 

73 

74 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

75 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

76 "chromatic corrections."), 

77 name="fgcmLookUpTable", 

78 storageClass="Catalog", 

79 dimensions=("instrument",), 

80 deferLoad=True, 

81 ) 

82 

83 fgcmVisitCatalog = connectionTypes.Input( 

84 doc="Catalog of visit information for fgcm", 

85 name="fgcmVisitCatalog", 

86 storageClass="Catalog", 

87 dimensions=("instrument",), 

88 deferLoad=True, 

89 ) 

90 

91 fgcmStarObservations = connectionTypes.Input( 

92 doc="Catalog of star observations for fgcm", 

93 name="fgcmStarObservations", 

94 storageClass="Catalog", 

95 dimensions=("instrument",), 

96 deferLoad=True, 

97 ) 

98 

99 fgcmStarIds = connectionTypes.Input( 

100 doc="Catalog of fgcm calibration star IDs", 

101 name="fgcmStarIds", 

102 storageClass="Catalog", 

103 dimensions=("instrument",), 

104 deferLoad=True, 

105 ) 

106 

107 fgcmStarIndices = connectionTypes.Input( 

108 doc="Catalog of fgcm calibration star indices", 

109 name="fgcmStarIndices", 

110 storageClass="Catalog", 

111 dimensions=("instrument",), 

112 deferLoad=True, 

113 ) 

114 

115 fgcmReferenceStars = connectionTypes.Input( 

116 doc="Catalog of fgcm-matched reference stars", 

117 name="fgcmReferenceStars", 

118 storageClass="Catalog", 

119 dimensions=("instrument",), 

120 deferLoad=True, 

121 ) 

122 

123 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput( 

124 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle", 

125 name="fgcmFlaggedStars{previousCycleNumber}", 

126 storageClass="Catalog", 

127 dimensions=("instrument",), 

128 deferLoad=True, 

129 ) 

130 

131 fgcmFitParametersInput = connectionTypes.PrerequisiteInput( 

132 doc="Catalog of fgcm fit parameters from previous fit cycle", 

133 name="fgcmFitParameters{previousCycleNumber}", 

134 storageClass="Catalog", 

135 dimensions=("instrument",), 

136 deferLoad=True, 

137 ) 

138 

139 fgcmFitParameters = connectionTypes.Output( 

140 doc="Catalog of fgcm fit parameters from current fit cycle", 

141 name="fgcmFitParameters{cycleNumber}", 

142 storageClass="Catalog", 

143 dimensions=("instrument",), 

144 ) 

145 

146 fgcmFlaggedStars = connectionTypes.Output( 

147 doc="Catalog of flagged stars for fgcm calibration from current fit cycle", 

148 name="fgcmFlaggedStars{cycleNumber}", 

149 storageClass="Catalog", 

150 dimensions=("instrument",), 

151 ) 

152 

153 fgcmZeropoints = connectionTypes.Output( 

154 doc="Catalog of fgcm zeropoint data from current fit cycle", 

155 name="fgcmZeropoints{cycleNumber}", 

156 storageClass="Catalog", 

157 dimensions=("instrument",), 

158 ) 

159 

160 fgcmAtmosphereParameters = connectionTypes.Output( 

161 doc="Catalog of atmospheric fit parameters from current fit cycle", 

162 name="fgcmAtmosphereParameters{cycleNumber}", 

163 storageClass="Catalog", 

164 dimensions=("instrument",), 

165 ) 

166 

167 fgcmStandardStars = connectionTypes.Output( 

168 doc="Catalog of standard star magnitudes from current fit cycle", 

169 name="fgcmStandardStars{cycleNumber}", 

170 storageClass="SimpleCatalog", 

171 dimensions=("instrument",), 

172 ) 

173 

174 # Add connections for running multiple cycles 

175 # This uses vars() to alter the class __dict__ to programmatically 

176 # write many similar outputs. 

177 for cycle in range(MULTIPLE_CYCLES_MAX): 

178 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output( 

179 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}", 

180 name=f"fgcmFitParameters{cycle}", 

181 storageClass="Catalog", 

182 dimensions=("instrument",), 

183 ) 

184 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output( 

185 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}", 

186 name=f"fgcmFlaggedStars{cycle}", 

187 storageClass="Catalog", 

188 dimensions=("instrument",), 

189 ) 

190 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output( 

191 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}", 

192 name=f"fgcmZeropoints{cycle}", 

193 storageClass="Catalog", 

194 dimensions=("instrument",), 

195 ) 

196 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output( 

197 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}", 

198 name=f"fgcmAtmosphereParameters{cycle}", 

199 storageClass="Catalog", 

200 dimensions=("instrument",), 

201 ) 

202 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output( 

203 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}", 

204 name=f"fgcmStandardStars{cycle}", 

205 storageClass="SimpleCatalog", 

206 dimensions=("instrument",), 

207 ) 

208 

209 def __init__(self, *, config=None): 

210 super().__init__(config=config) 

211 

212 if not config.doReferenceCalibration: 

213 self.inputs.remove("fgcmReferenceStars") 

214 

215 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber: 

216 raise ValueError("cycleNumber must be of integer format") 

217 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber: 

218 raise ValueError("previousCycleNumber must be of integer format") 

219 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1): 

220 raise ValueError("previousCycleNumber must be 1 less than cycleNumber") 

221 

222 if int(config.connections.cycleNumber) == 0: 

223 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput") 

224 self.prerequisiteInputs.remove("fgcmFitParametersInput") 

225 

226 if not self.config.doMultipleCycles: 

227 # Single-cycle run 

228 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle: 

229 self.outputs.remove("fgcmStandardStars") 

230 

231 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle: 

232 self.outputs.remove("fgcmZeropoints") 

233 self.outputs.remove("fgcmAtmosphereParameters") 

234 

235 # Remove all multiple cycle outputs 

236 for cycle in range(0, MULTIPLE_CYCLES_MAX): 

237 self.outputs.remove(f"fgcmFitParameters{cycle}") 

238 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

239 self.outputs.remove(f"fgcmZeropoints{cycle}") 

240 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

241 self.outputs.remove(f"fgcmStandardStars{cycle}") 

242 

243 else: 

244 # Multiple-cycle run 

245 # Remove single-cycle outputs 

246 self.outputs.remove("fgcmFitParameters") 

247 self.outputs.remove("fgcmFlaggedStars") 

248 self.outputs.remove("fgcmZeropoints") 

249 self.outputs.remove("fgcmAtmosphereParameters") 

250 self.outputs.remove("fgcmStandardStars") 

251 

252 # Remove outputs from cycles that are not used 

253 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1, 

254 MULTIPLE_CYCLES_MAX): 

255 self.outputs.remove(f"fgcmFitParameters{cycle}") 

256 self.outputs.remove(f"fgcmFlaggedStars{cycle}") 

257 self.outputs.remove(f"fgcmZeropoints{cycle}") 

258 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

259 self.outputs.remove(f"fgcmStandardStars{cycle}") 

260 

261 # Remove non-final-cycle outputs if necessary 

262 for cycle in range(self.config.multipleCyclesFinalCycleNumber): 

263 if not self.config.outputZeropointsBeforeFinalCycle: 

264 self.outputs.remove(f"fgcmZeropoints{cycle}") 

265 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}") 

266 if not self.config.outputStandardsBeforeFinalCycle: 

267 self.outputs.remove(f"fgcmStandardStars{cycle}") 

268 

269 

270class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig, 

271 pipelineConnections=FgcmFitCycleConnections): 

272 """Config for FgcmFitCycle""" 

273 

274 doMultipleCycles = pexConfig.Field( 

275 doc="Run multiple fit cycles in one task", 

276 dtype=bool, 

277 default=False, 

278 ) 

279 multipleCyclesFinalCycleNumber = pexConfig.RangeField( 

280 doc=("Final cycle number in multiple cycle mode. The initial cycle " 

281 "is 0, with limited parameters fit. The next cycle is 1 with " 

282 "full parameter fit. The final cycle is a clean-up with no " 

283 "parameters fit. There will be a total of " 

284 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final " 

285 "cycle number cannot be less than 2."), 

286 dtype=int, 

287 default=5, 

288 min=2, 

289 max=MULTIPLE_CYCLES_MAX, 

290 inclusiveMax=True, 

291 ) 

292 bands = pexConfig.ListField( 

293 doc="Bands to run calibration", 

294 dtype=str, 

295 default=[], 

296 ) 

297 fitBands = pexConfig.ListField( 

298 doc=("Bands to use in atmospheric fit. The bands not listed here will have " 

299 "the atmosphere constrained from the 'fitBands' on the same night. " 

300 "Must be a subset of `config.bands`"), 

301 dtype=str, 

302 default=[], 

303 ) 

304 requiredBands = pexConfig.ListField( 

305 doc=("Bands that are required for a star to be considered a calibration star. " 

306 "Must be a subset of `config.bands`"), 

307 dtype=str, 

308 default=[], 

309 ) 

310 # The following config will not be necessary after Gen2 retirement. 

311 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which 

312 # is easiest to access in the config file. 

313 physicalFilterMap = pexConfig.DictField( 

314 doc="Mapping from 'physicalFilter' to band.", 

315 keytype=str, 

316 itemtype=str, 

317 default={}, 

318 ) 

319 doReferenceCalibration = pexConfig.Field( 

320 doc="Use reference catalog as additional constraint on calibration", 

321 dtype=bool, 

322 default=True, 

323 ) 

324 refStarSnMin = pexConfig.Field( 

325 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.", 

326 dtype=float, 

327 default=50.0, 

328 ) 

329 refStarOutlierNSig = pexConfig.Field( 

330 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. " 

331 "Computed per-band, and if it is an outlier in any band it is rejected from fits."), 

332 dtype=float, 

333 default=4.0, 

334 ) 

335 applyRefStarColorCuts = pexConfig.Field( 

336 doc="Apply color cuts to reference stars?", 

337 dtype=bool, 

338 default=True, 

339 ) 

340 nCore = pexConfig.Field( 

341 doc="Number of cores to use", 

342 dtype=int, 

343 default=4, 

344 ) 

345 nStarPerRun = pexConfig.Field( 

346 doc="Number of stars to run in each chunk", 

347 dtype=int, 

348 default=200000, 

349 ) 

350 nExpPerRun = pexConfig.Field( 

351 doc="Number of exposures to run in each chunk", 

352 dtype=int, 

353 default=1000, 

354 ) 

355 reserveFraction = pexConfig.Field( 

356 doc="Fraction of stars to reserve for testing", 

357 dtype=float, 

358 default=0.1, 

359 ) 

360 freezeStdAtmosphere = pexConfig.Field( 

361 doc="Freeze atmosphere parameters to standard (for testing)", 

362 dtype=bool, 

363 default=False, 

364 ) 

365 precomputeSuperStarInitialCycle = pexConfig.Field( 

366 doc="Precompute superstar flat for initial cycle", 

367 dtype=bool, 

368 default=False, 

369 ) 

370 superStarSubCcdDict = pexConfig.DictField( 

371 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. " 

372 "Must have one entry per band."), 

373 keytype=str, 

374 itemtype=bool, 

375 default={}, 

376 ) 

377 superStarSubCcdChebyshevOrder = pexConfig.Field( 

378 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 

379 "Global default is first-order polynomials, and should be overridden " 

380 "on a camera-by-camera basis depending on the ISR."), 

381 dtype=int, 

382 default=1, 

383 ) 

384 superStarSubCcdTriangular = pexConfig.Field( 

385 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to " 

386 "suppress high-order cross terms?"), 

387 dtype=bool, 

388 default=False, 

389 ) 

390 superStarSigmaClip = pexConfig.Field( 

391 doc="Number of sigma to clip outliers when selecting for superstar flats", 

392 dtype=float, 

393 default=5.0, 

394 ) 

395 focalPlaneSigmaClip = pexConfig.Field( 

396 doc="Number of sigma to clip outliers per focal-plane.", 

397 dtype=float, 

398 default=4.0, 

399 ) 

400 ccdGraySubCcdDict = pexConfig.DictField( 

401 doc=("Per-band specification on whether to compute achromatic per-ccd residual " 

402 "('ccd gray') on a sub-ccd scale."), 

403 keytype=str, 

404 itemtype=bool, 

405 default={}, 

406 ) 

407 ccdGraySubCcdChebyshevOrder = pexConfig.Field( 

408 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.", 

409 dtype=int, 

410 default=1, 

411 ) 

412 ccdGraySubCcdTriangular = pexConfig.Field( 

413 doc=("Should the sub-ccd gray chebyshev matrix be triangular to " 

414 "suppress high-order cross terms?"), 

415 dtype=bool, 

416 default=True, 

417 ) 

418 ccdGrayFocalPlaneDict = pexConfig.DictField( 

419 doc=("Per-band specification on whether to compute focal-plane residual " 

420 "('ccd gray') corrections."), 

421 keytype=str, 

422 itemtype=bool, 

423 default={}, 

424 ) 

425 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field( 

426 doc=("Minimum number of 'good' CCDs required to perform focal-plane " 

427 "gray corrections. If there are fewer good CCDs then the gray " 

428 "correction is computed per-ccd."), 

429 dtype=int, 

430 default=1, 

431 ) 

432 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field( 

433 doc="Order of the 2D chebyshev polynomials for focal plane fit.", 

434 dtype=int, 

435 default=3, 

436 ) 

437 cycleNumber = pexConfig.Field( 

438 doc=("FGCM fit cycle number. This is automatically incremented after each run " 

439 "and stage of outlier rejection. See cookbook for details."), 

440 dtype=int, 

441 default=None, 

442 ) 

443 isFinalCycle = pexConfig.Field( 

444 doc=("Is this the final cycle of the fitting? Will automatically compute final " 

445 "selection of stars and photometric exposures, and will output zeropoints " 

446 "and standard stars for use in fgcmOutputProducts"), 

447 dtype=bool, 

448 default=False, 

449 ) 

450 maxIterBeforeFinalCycle = pexConfig.Field( 

451 doc=("Maximum fit iterations, prior to final cycle. The number of iterations " 

452 "will always be 0 in the final cycle for cleanup and final selection."), 

453 dtype=int, 

454 default=50, 

455 ) 

456 deltaMagBkgOffsetPercentile = pexConfig.Field( 

457 doc=("Percentile brightest stars on a visit/ccd to use to compute net " 

458 "offset from local background subtraction."), 

459 dtype=float, 

460 default=0.25, 

461 ) 

462 deltaMagBkgPerCcd = pexConfig.Field( 

463 doc=("Compute net offset from local background subtraction per-ccd? " 

464 "Otherwise, use computation per visit."), 

465 dtype=bool, 

466 default=False, 

467 ) 

468 utBoundary = pexConfig.Field( 

469 doc="Boundary (in UTC) from day-to-day", 

470 dtype=float, 

471 default=None, 

472 ) 

473 washMjds = pexConfig.ListField( 

474 doc="Mirror wash MJDs", 

475 dtype=float, 

476 default=(0.0,), 

477 ) 

478 epochMjds = pexConfig.ListField( 

479 doc="Epoch boundaries in MJD", 

480 dtype=float, 

481 default=(0.0,), 

482 ) 

483 minObsPerBand = pexConfig.Field( 

484 doc="Minimum good observations per band", 

485 dtype=int, 

486 default=2, 

487 ) 

488 # TODO: When DM-16511 is done, it will be possible to get the 

489 # telescope latitude directly from the camera. 

490 latitude = pexConfig.Field( 

491 doc="Observatory latitude", 

492 dtype=float, 

493 default=None, 

494 ) 

495 brightObsGrayMax = pexConfig.Field( 

496 doc="Maximum gray extinction to be considered bright observation", 

497 dtype=float, 

498 default=0.15, 

499 ) 

500 minStarPerCcd = pexConfig.Field( 

501 doc=("Minimum number of good stars per CCD to be used in calibration fit. " 

502 "CCDs with fewer stars will have their calibration estimated from other " 

503 "CCDs in the same visit, with zeropoint error increased accordingly."), 

504 dtype=int, 

505 default=5, 

506 ) 

507 minCcdPerExp = pexConfig.Field( 

508 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 

509 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."), 

510 dtype=int, 

511 default=5, 

512 ) 

513 maxCcdGrayErr = pexConfig.Field( 

514 doc="Maximum error on CCD gray offset to be considered photometric", 

515 dtype=float, 

516 default=0.05, 

517 ) 

518 minStarPerExp = pexConfig.Field( 

519 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. " 

520 "Visits with fewer good stars will have CCD zeropoints estimated where possible."), 

521 dtype=int, 

522 default=600, 

523 ) 

524 minExpPerNight = pexConfig.Field( 

525 doc="Minimum number of good exposures/visits to consider a partly photometric night", 

526 dtype=int, 

527 default=10, 

528 ) 

529 expGrayInitialCut = pexConfig.Field( 

530 doc=("Maximum exposure/visit gray value for initial selection of possible photometric " 

531 "observations."), 

532 dtype=float, 

533 default=-0.25, 

534 ) 

535 expGrayPhotometricCutDict = pexConfig.DictField( 

536 doc=("Per-band specification on maximum (negative) achromatic exposure residual " 

537 "('gray term') for a visit to be considered photometric. Must have one " 

538 "entry per band. Broad-band filters should be -0.05."), 

539 keytype=str, 

540 itemtype=float, 

541 default={}, 

542 ) 

543 expGrayHighCutDict = pexConfig.DictField( 

544 doc=("Per-band specification on maximum (positive) achromatic exposure residual " 

545 "('gray term') for a visit to be considered photometric. Must have one " 

546 "entry per band. Broad-band filters should be 0.2."), 

547 keytype=str, 

548 itemtype=float, 

549 default={}, 

550 ) 

551 expGrayRecoverCut = pexConfig.Field( 

552 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 

553 "Visits with more gray extinction will only get CCD zeropoints if there are " 

554 "sufficient star observations (minStarPerCcd) on that CCD."), 

555 dtype=float, 

556 default=-1.0, 

557 ) 

558 expVarGrayPhotometricCutDict = pexConfig.DictField( 

559 doc=("Per-band specification on maximum exposure variance to be considered possibly " 

560 "photometric. Must have one entry per band. Broad-band filters should be " 

561 "0.0005."), 

562 keytype=str, 

563 itemtype=float, 

564 default={}, 

565 ) 

566 expGrayErrRecoverCut = pexConfig.Field( 

567 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. " 

568 "Visits with more gray variance will only get CCD zeropoints if there are " 

569 "sufficient star observations (minStarPerCcd) on that CCD."), 

570 dtype=float, 

571 default=0.05, 

572 ) 

573 aperCorrFitNBins = pexConfig.Field( 

574 doc=("Number of aperture bins used in aperture correction fit. When set to 0" 

575 "no fit will be performed, and the config.aperCorrInputSlopes will be " 

576 "used if available."), 

577 dtype=int, 

578 default=10, 

579 ) 

580 aperCorrInputSlopeDict = pexConfig.DictField( 

581 doc=("Per-band specification of aperture correction input slope parameters. These " 

582 "are used on the first fit iteration, and aperture correction parameters will " 

583 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 

584 "to set this when there is insufficient data to fit the parameters (e.g. " 

585 "tract mode)."), 

586 keytype=str, 

587 itemtype=float, 

588 default={}, 

589 ) 

590 sedboundaryterms = pexConfig.ConfigField( 

591 doc="Mapping from bands to SED boundary term names used is sedterms.", 

592 dtype=SedboundarytermDict, 

593 ) 

594 sedterms = pexConfig.ConfigField( 

595 doc="Mapping from terms to bands for fgcm linear SED approximations.", 

596 dtype=SedtermDict, 

597 ) 

598 sigFgcmMaxErr = pexConfig.Field( 

599 doc="Maximum mag error for fitting sigma_FGCM", 

600 dtype=float, 

601 default=0.01, 

602 ) 

603 sigFgcmMaxEGrayDict = pexConfig.DictField( 

604 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) " 

605 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 

606 "should be 0.05."), 

607 keytype=str, 

608 itemtype=float, 

609 default={}, 

610 ) 

611 ccdGrayMaxStarErr = pexConfig.Field( 

612 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) " 

613 "computation"), 

614 dtype=float, 

615 default=0.10, 

616 ) 

617 approxThroughputDict = pexConfig.DictField( 

618 doc=("Per-band specification of the approximate overall throughput at the start of " 

619 "calibration observations. Must have one entry per band. Typically should " 

620 "be 1.0."), 

621 keytype=str, 

622 itemtype=float, 

623 default={}, 

624 ) 

625 sigmaCalRange = pexConfig.ListField( 

626 doc="Allowed range for systematic error floor estimation", 

627 dtype=float, 

628 default=(0.001, 0.003), 

629 ) 

630 sigmaCalFitPercentile = pexConfig.ListField( 

631 doc="Magnitude percentile range to fit systematic error floor", 

632 dtype=float, 

633 default=(0.05, 0.15), 

634 ) 

635 sigmaCalPlotPercentile = pexConfig.ListField( 

636 doc="Magnitude percentile range to plot systematic error floor", 

637 dtype=float, 

638 default=(0.05, 0.95), 

639 ) 

640 sigma0Phot = pexConfig.Field( 

641 doc="Systematic error floor for all zeropoints", 

642 dtype=float, 

643 default=0.003, 

644 ) 

645 mapLongitudeRef = pexConfig.Field( 

646 doc="Reference longitude for plotting maps", 

647 dtype=float, 

648 default=0.0, 

649 ) 

650 mapNSide = pexConfig.Field( 

651 doc="Healpix nside for plotting maps", 

652 dtype=int, 

653 default=256, 

654 ) 

655 outfileBase = pexConfig.Field( 

656 doc="Filename start for plot output files", 

657 dtype=str, 

658 default=None, 

659 ) 

660 starColorCuts = pexConfig.ListField( 

661 doc="Encoded star-color cuts (to be cleaned up)", 

662 dtype=str, 

663 default=("NO_DATA",), 

664 ) 

665 colorSplitBands = pexConfig.ListField( 

666 doc="Band names to use to split stars by color. Must have 2 entries.", 

667 dtype=str, 

668 length=2, 

669 default=('g', 'i'), 

670 ) 

671 modelMagErrors = pexConfig.Field( 

672 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)", 

673 dtype=bool, 

674 default=True, 

675 ) 

676 useQuadraticPwv = pexConfig.Field( 

677 doc="Model PWV with a quadratic term for variation through the night?", 

678 dtype=bool, 

679 default=False, 

680 ) 

681 instrumentParsPerBand = pexConfig.Field( 

682 doc=("Model instrumental parameters per band? " 

683 "Otherwise, instrumental parameters (QE changes with time) are " 

684 "shared among all bands."), 

685 dtype=bool, 

686 default=False, 

687 ) 

688 instrumentSlopeMinDeltaT = pexConfig.Field( 

689 doc=("Minimum time change (in days) between observations to use in constraining " 

690 "instrument slope."), 

691 dtype=float, 

692 default=20.0, 

693 ) 

694 fitMirrorChromaticity = pexConfig.Field( 

695 doc="Fit (intraband) mirror chromatic term?", 

696 dtype=bool, 

697 default=False, 

698 ) 

699 coatingMjds = pexConfig.ListField( 

700 doc="Mirror coating dates in MJD", 

701 dtype=float, 

702 default=(0.0,), 

703 ) 

704 outputStandardsBeforeFinalCycle = pexConfig.Field( 

705 doc="Output standard stars prior to final cycle? Used in debugging.", 

706 dtype=bool, 

707 default=False, 

708 ) 

709 outputZeropointsBeforeFinalCycle = pexConfig.Field( 

710 doc="Output standard stars prior to final cycle? Used in debugging.", 

711 dtype=bool, 

712 default=False, 

713 ) 

714 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField( 

715 doc=("Per-band specification on whether to use star repeatability (instead of exposures) " 

716 "for computing photometric cuts. Recommended for tract mode or bands with few visits."), 

717 keytype=str, 

718 itemtype=bool, 

719 default={}, 

720 ) 

721 autoPhotometricCutNSig = pexConfig.Field( 

722 doc=("Number of sigma for automatic computation of (low) photometric cut. " 

723 "Cut is based on exposure gray width (per band), unless " 

724 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

725 "repeatability is used (also per band)."), 

726 dtype=float, 

727 default=3.0, 

728 ) 

729 autoHighCutNSig = pexConfig.Field( 

730 doc=("Number of sigma for automatic computation of (high) outlier cut. " 

731 "Cut is based on exposure gray width (per band), unless " 

732 "useRepeatabilityForExpGrayCuts is set, in which case the star " 

733 "repeatability is used (also per band)."), 

734 dtype=float, 

735 default=4.0, 

736 ) 

737 quietMode = pexConfig.Field( 

738 doc="Be less verbose with logging.", 

739 dtype=bool, 

740 default=False, 

741 ) 

742 doPlots = pexConfig.Field( 

743 doc="Make fgcm QA plots.", 

744 dtype=bool, 

745 default=True, 

746 ) 

747 randomSeed = pexConfig.Field( 

748 doc="Random seed for fgcm for consistency in tests.", 

749 dtype=int, 

750 default=None, 

751 optional=True, 

752 ) 

753 

754 def validate(self): 

755 super().validate() 

756 

757 if self.connections.previousCycleNumber != str(self.cycleNumber - 1): 

758 msg = "cycleNumber in template must be connections.previousCycleNumber + 1" 

759 raise RuntimeError(msg) 

760 if self.connections.cycleNumber != str(self.cycleNumber): 

761 msg = "cycleNumber in template must be equal to connections.cycleNumber" 

762 raise RuntimeError(msg) 

763 

764 for band in self.fitBands: 

765 if band not in self.bands: 

766 msg = 'fitBand %s not in bands' % (band) 

767 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg) 

768 for band in self.requiredBands: 

769 if band not in self.bands: 

770 msg = 'requiredBand %s not in bands' % (band) 

771 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg) 

772 for band in self.colorSplitBands: 

773 if band not in self.bands: 

774 msg = 'colorSplitBand %s not in bands' % (band) 

775 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg) 

776 for band in self.bands: 

777 if band not in self.superStarSubCcdDict: 

778 msg = 'band %s not in superStarSubCcdDict' % (band) 

779 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict, 

780 self, msg) 

781 if band not in self.ccdGraySubCcdDict: 

782 msg = 'band %s not in ccdGraySubCcdDict' % (band) 

783 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict, 

784 self, msg) 

785 if band not in self.expGrayPhotometricCutDict: 

786 msg = 'band %s not in expGrayPhotometricCutDict' % (band) 

787 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict, 

788 self, msg) 

789 if band not in self.expGrayHighCutDict: 

790 msg = 'band %s not in expGrayHighCutDict' % (band) 

791 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict, 

792 self, msg) 

793 if band not in self.expVarGrayPhotometricCutDict: 

794 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band) 

795 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict, 

796 self, msg) 

797 if band not in self.sigFgcmMaxEGrayDict: 

798 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band) 

799 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict, 

800 self, msg) 

801 if band not in self.approxThroughputDict: 

802 msg = 'band %s not in approxThroughputDict' % (band) 

803 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict, 

804 self, msg) 

805 if band not in self.useRepeatabilityForExpGrayCutsDict: 

806 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band) 

807 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict, 

808 self, msg) 

809 

810 

811class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner): 

812 """Subclass of TaskRunner for fgcmFitCycleTask 

813 

814 fgcmFitCycleTask.run() takes one argument, the butler, and uses 

815 stars and visits previously extracted from dataRefs by 

816 fgcmBuildStars. 

817 This Runner does not perform any dataRef parallelization, but the FGCM 

818 code called by the Task uses python multiprocessing (see the "ncores" 

819 config option). 

820 """ 

821 

822 @staticmethod 

823 def getTargetList(parsedCmd): 

824 """ 

825 Return a list with one element, the butler. 

826 """ 

827 return [parsedCmd.butler] 

828 

829 def __call__(self, butler): 

830 """ 

831 Parameters 

832 ---------- 

833 butler: `lsst.daf.persistence.Butler` 

834 

835 Returns 

836 ------- 

837 exitStatus: `list` with `pipeBase.Struct` 

838 exitStatus (0: success; 1: failure) 

839 """ 

840 

841 task = self.TaskClass(config=self.config, log=self.log) 

842 

843 exitStatus = 0 

844 if self.doRaise: 

845 task.runDataRef(butler) 

846 else: 

847 try: 

848 task.runDataRef(butler) 

849 except Exception as e: 

850 exitStatus = 1 

851 task.log.fatal("Failed: %s" % e) 

852 if not isinstance(e, pipeBase.TaskError): 

853 traceback.print_exc(file=sys.stderr) 

854 

855 task.writeMetadata(butler) 

856 

857 # The task does not return any results: 

858 return [pipeBase.Struct(exitStatus=exitStatus)] 

859 

860 def run(self, parsedCmd): 

861 """ 

862 Run the task, with no multiprocessing 

863 

864 Parameters 

865 ---------- 

866 parsedCmd: ArgumentParser parsed command line 

867 """ 

868 

869 resultList = [] 

870 

871 if self.precall(parsedCmd): 

872 targetList = self.getTargetList(parsedCmd) 

873 # make sure that we only get 1 

874 resultList = self(targetList[0]) 

875 

876 return resultList 

877 

878 

879class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

880 """ 

881 Run Single fit cycle for FGCM global calibration 

882 """ 

883 

884 ConfigClass = FgcmFitCycleConfig 

885 RunnerClass = FgcmFitCycleRunner 

886 _DefaultName = "fgcmFitCycle" 

887 

888 def __init__(self, butler=None, initInputs=None, **kwargs): 

889 super().__init__(**kwargs) 

890 

891 # no saving of metadata for now 

892 def _getMetadataName(self): 

893 return None 

894 

895 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

896 camera = butlerQC.get(inputRefs.camera) 

897 

898 dataRefDict = {} 

899 

900 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) 

901 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog) 

902 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations) 

903 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds) 

904 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices) 

905 if self.config.doReferenceCalibration: 

906 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars) 

907 if self.config.cycleNumber > 0: 

908 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput) 

909 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput) 

910 

911 fgcmDatasetDict = None 

912 if self.config.doMultipleCycles: 

913 # Run multiple cycles at once. 

914 config = copy.copy(self.config) 

915 config.update(cycleNumber=0) 

916 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1): 

917 if cycle == self.config.multipleCyclesFinalCycleNumber: 

918 config.update(isFinalCycle=True) 

919 

920 if cycle > 0: 

921 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars'] 

922 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters'] 

923 

924 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config) 

925 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], 

926 getattr(outputRefs, f'fgcmFitParameters{cycle}')) 

927 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], 

928 getattr(outputRefs, f'fgcmFlaggedStars{cycle}')) 

929 if self.outputZeropoints: 

930 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], 

931 getattr(outputRefs, f'fgcmZeropoints{cycle}')) 

932 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 

933 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}')) 

934 if self.outputStandards: 

935 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], 

936 getattr(outputRefs, f'fgcmStandardStars{cycle}')) 

937 else: 

938 # Run a single cycle 

939 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict) 

940 

941 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters) 

942 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars) 

943 if self.outputZeropoints: 

944 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints) 

945 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters) 

946 if self.outputStandards: 

947 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars) 

948 

949 @pipeBase.timeMethod 

950 def runDataRef(self, butler): 

951 """ 

952 Run a single fit cycle for FGCM 

953 

954 Parameters 

955 ---------- 

956 butler: `lsst.daf.persistence.Butler` 

957 """ 

958 self._checkDatasetsExist(butler) 

959 

960 dataRefDict = {} 

961 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable') 

962 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog') 

963 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations') 

964 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds') 

965 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices') 

966 if self.config.doReferenceCalibration: 

967 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars') 

968 if self.config.cycleNumber > 0: 

969 lastCycle = self.config.cycleNumber - 1 

970 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars', 

971 fgcmcycle=lastCycle) 

972 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters', 

973 fgcmcycle=lastCycle) 

974 

975 camera = butler.get('camera') 

976 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict) 

977 

978 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters', 

979 fgcmcycle=self.config.cycleNumber) 

980 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars', 

981 fgcmcycle=self.config.cycleNumber) 

982 if self.outputZeropoints: 

983 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints', 

984 fgcmcycle=self.config.cycleNumber) 

985 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters', 

986 fgcmcycle=self.config.cycleNumber) 

987 if self.outputStandards: 

988 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars', 

989 fgcmcycle=self.config.cycleNumber) 

990 

991 def writeConfig(self, butler, clobber=False, doBackup=True): 

992 """Write the configuration used for processing the data, or check that an existing 

993 one is equal to the new one if present. This is an override of the regular 

994 version from pipe_base that knows about fgcmcycle. 

995 

996 Parameters 

997 ---------- 

998 butler : `lsst.daf.persistence.Butler` 

999 Data butler used to write the config. The config is written to dataset type 

1000 `CmdLineTask._getConfigName`. 

1001 clobber : `bool`, optional 

1002 A boolean flag that controls what happens if a config already has been saved: 

1003 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 

1004 - `False`: raise `TaskError` if this config does not match the existing config. 

1005 doBackup : `bool`, optional 

1006 Set to `True` to backup the config files if clobbering. 

1007 """ 

1008 configName = self._getConfigName() 

1009 if configName is None: 

1010 return 

1011 if clobber: 

1012 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber) 

1013 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber): 

1014 # this may be subject to a race condition; see #2789 

1015 try: 

1016 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber) 

1017 except Exception as exc: 

1018 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" % 

1019 (configName, exc)) 

1020 

1021 def logConfigMismatch(msg): 

1022 self.log.fatal("Comparing configuration: %s", msg) 

1023 

1024 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

1025 raise pipeBase.TaskError( 

1026 f"Config does not match existing task config {configName!r} on disk; tasks configurations" 

1027 " must be consistent within the same output repo (override with --clobber-config)") 

1028 else: 

1029 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber) 

1030 

1031 def _fgcmFitCycle(self, camera, dataRefDict, config=None): 

1032 """ 

1033 Run the fit cycle 

1034 

1035 Parameters 

1036 ---------- 

1037 camera : `lsst.afw.cameraGeom.Camera` 

1038 dataRefDict : `dict` 

1039 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or 

1040 `lsst.daf.butler.DeferredDatasetHandle` (gen3) 

1041 dataRef dictionary with keys: 

1042 

1043 ``"fgcmLookUpTable"`` 

1044 dataRef for the FGCM look-up table. 

1045 ``"fgcmVisitCatalog"`` 

1046 dataRef for visit summary catalog. 

1047 ``"fgcmStarObservations"`` 

1048 dataRef for star observation catalog. 

1049 ``"fgcmStarIds"`` 

1050 dataRef for star id catalog. 

1051 ``"fgcmStarIndices"`` 

1052 dataRef for star index catalog. 

1053 ``"fgcmReferenceStars"`` 

1054 dataRef for matched reference star catalog. 

1055 ``"fgcmFlaggedStars"`` 

1056 dataRef for flagged star catalog. 

1057 ``"fgcmFitParameters"`` 

1058 dataRef for fit parameter catalog. 

1059 config : `lsst.pex.config.Config`, optional 

1060 Configuration to use to override self.config. 

1061 

1062 Returns 

1063 ------- 

1064 fgcmDatasetDict : `dict` 

1065 Dictionary of datasets to persist. 

1066 """ 

1067 if config is not None: 

1068 _config = config 

1069 else: 

1070 _config = self.config 

1071 

1072 # Set defaults on whether to output standards and zeropoints 

1073 self.maxIter = _config.maxIterBeforeFinalCycle 

1074 self.outputStandards = _config.outputStandardsBeforeFinalCycle 

1075 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle 

1076 self.resetFitParameters = True 

1077 

1078 if _config.isFinalCycle: 

1079 # This is the final fit cycle, so we do not want to reset fit 

1080 # parameters, we want to run a final "clean-up" with 0 fit iterations, 

1081 # and we always want to output standards and zeropoints 

1082 self.maxIter = 0 

1083 self.outputStandards = True 

1084 self.outputZeropoints = True 

1085 self.resetFitParameters = False 

1086 

1087 lutCat = dataRefDict['fgcmLookUpTable'].get() 

1088 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, 

1089 dict(_config.physicalFilterMap)) 

1090 del lutCat 

1091 

1092 configDict = makeConfigDict(_config, self.log, camera, 

1093 self.maxIter, self.resetFitParameters, 

1094 self.outputZeropoints, 

1095 lutIndexVals[0]['FILTERNAMES']) 

1096 

1097 # next we need the exposure/visit information 

1098 visitCat = dataRefDict['fgcmVisitCatalog'].get() 

1099 fgcmExpInfo = translateVisitCatalog(visitCat) 

1100 del visitCat 

1101 

1102 # Use the first orientation. 

1103 # TODO: DM-21215 will generalize to arbitrary camera orientations 

1104 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0]) 

1105 

1106 noFitsDict = {'lutIndex': lutIndexVals, 

1107 'lutStd': lutStd, 

1108 'expInfo': fgcmExpInfo, 

1109 'ccdOffsets': ccdOffsets} 

1110 

1111 # set up the fitter object 

1112 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False, 

1113 noFitsDict=noFitsDict, noOutput=True) 

1114 

1115 # create the parameter object 

1116 if (fgcmFitCycle.initialCycle): 

1117 # cycle = 0, initial cycle 

1118 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig, 

1119 fgcmLut, 

1120 fgcmExpInfo) 

1121 else: 

1122 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog): 

1123 parCat = dataRefDict['fgcmFitParameters'] 

1124 else: 

1125 parCat = dataRefDict['fgcmFitParameters'].get() 

1126 inParInfo, inParams, inSuperStar = self._loadParameters(parCat) 

1127 del parCat 

1128 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig, 

1129 fgcmExpInfo, 

1130 inParInfo, 

1131 inParams, 

1132 inSuperStar) 

1133 

1134 # set up the stars... 

1135 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig) 

1136 

1137 starObs = dataRefDict['fgcmStarObservations'].get() 

1138 starIds = dataRefDict['fgcmStarIds'].get() 

1139 starIndices = dataRefDict['fgcmStarIndices'].get() 

1140 

1141 # grab the flagged stars if available 

1142 if 'fgcmFlaggedStars' in dataRefDict: 

1143 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog): 

1144 flaggedStars = dataRefDict['fgcmFlaggedStars'] 

1145 else: 

1146 flaggedStars = dataRefDict['fgcmFlaggedStars'].get() 

1147 flagId = flaggedStars['objId'][:] 

1148 flagFlag = flaggedStars['objFlag'][:] 

1149 else: 

1150 flaggedStars = None 

1151 flagId = None 

1152 flagFlag = None 

1153 

1154 if _config.doReferenceCalibration: 

1155 refStars = dataRefDict['fgcmReferenceStars'].get() 

1156 

1157 refMag, refMagErr = extractReferenceMags(refStars, 

1158 _config.bands, 

1159 _config.physicalFilterMap) 

1160 refId = refStars['fgcm_id'][:] 

1161 else: 

1162 refStars = None 

1163 refId = None 

1164 refMag = None 

1165 refMagErr = None 

1166 

1167 # match star observations to visits 

1168 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will 

1169 # actually be transferred into fgcm using the indexing below. 

1170 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']]) 

1171 

1172 # The fgcmStars.loadStars method will copy all the star information into 

1173 # special shared memory objects that will not blow up the memory usage when 

1174 # used with python multiprocessing. Once all the numbers are copied, 

1175 # it is necessary to release all references to the objects that previously 

1176 # stored the data to ensure that the garbage collector can clear the memory, 

1177 # and ensure that this memory is not copied when multiprocessing kicks in. 

1178 

1179 # We determine the conversion from the native units (typically radians) to 

1180 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

1181 # numpy arrays rather than Angles, which would we approximately 600x slower. 

1182 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra']) 

1183 

1184 fgcmStars.loadStars(fgcmPars, 

1185 starObs['visit'][starIndices['obsIndex']], 

1186 starObs['ccd'][starIndices['obsIndex']], 

1187 starObs['ra'][starIndices['obsIndex']] * conv, 

1188 starObs['dec'][starIndices['obsIndex']] * conv, 

1189 starObs['instMag'][starIndices['obsIndex']], 

1190 starObs['instMagErr'][starIndices['obsIndex']], 

1191 fgcmExpInfo['FILTERNAME'][visitIndex], 

1192 starIds['fgcm_id'][:], 

1193 starIds['ra'][:], 

1194 starIds['dec'][:], 

1195 starIds['obsArrIndex'][:], 

1196 starIds['nObs'][:], 

1197 obsX=starObs['x'][starIndices['obsIndex']], 

1198 obsY=starObs['y'][starIndices['obsIndex']], 

1199 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']], 

1200 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']], 

1201 refID=refId, 

1202 refMag=refMag, 

1203 refMagErr=refMagErr, 

1204 flagID=flagId, 

1205 flagFlag=flagFlag, 

1206 computeNobs=True) 

1207 

1208 # Release all references to temporary objects holding star data (see above) 

1209 del starObs 

1210 del starIds 

1211 del starIndices 

1212 del flagId 

1213 del flagFlag 

1214 del flaggedStars 

1215 del refStars 

1216 del refId 

1217 del refMag 

1218 del refMagErr 

1219 

1220 # and set the bits in the cycle object 

1221 fgcmFitCycle.setLUT(fgcmLut) 

1222 fgcmFitCycle.setStars(fgcmStars, fgcmPars) 

1223 fgcmFitCycle.setPars(fgcmPars) 

1224 

1225 # finish the setup 

1226 fgcmFitCycle.finishSetup() 

1227 

1228 # and run 

1229 fgcmFitCycle.run() 

1230 

1231 ################## 

1232 # Persistance 

1233 ################## 

1234 

1235 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle) 

1236 

1237 # Output the config for the next cycle 

1238 # We need to make a copy since the input one has been frozen 

1239 

1240 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for 

1241 i, b in enumerate(_config.bands)} 

1242 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for 

1243 i, band in enumerate(_config.bands)} 

1244 

1245 outConfig = copy.copy(_config) 

1246 outConfig.update(cycleNumber=(_config.cycleNumber + 1), 

1247 precomputeSuperStarInitialCycle=False, 

1248 freezeStdAtmosphere=False, 

1249 expGrayPhotometricCutDict=updatedPhotometricCutDict, 

1250 expGrayHighCutDict=updatedHighCutDict) 

1251 

1252 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber), 

1253 cycleNumber=str(_config.cycleNumber + 1)) 

1254 

1255 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase, 

1256 outConfig.cycleNumber) 

1257 outConfig.save(configFileName) 

1258 

1259 if _config.isFinalCycle == 1: 

1260 # We are done, ready to output products 

1261 self.log.info("Everything is in place to run fgcmOutputProducts.py") 

1262 else: 

1263 self.log.info("Saved config for next cycle to %s" % (configFileName)) 

1264 self.log.info("Be sure to look at:") 

1265 self.log.info(" config.expGrayPhotometricCut") 

1266 self.log.info(" config.expGrayHighCut") 

1267 self.log.info("If you are satisfied with the fit, please set:") 

1268 self.log.info(" config.isFinalCycle = True") 

1269 

1270 fgcmFitCycle.freeSharedMemory() 

1271 

1272 return fgcmDatasetDict, outConfig 

1273 

1274 def _checkDatasetsExist(self, butler): 

1275 """ 

1276 Check if necessary datasets exist to run fgcmFitCycle 

1277 

1278 Parameters 

1279 ---------- 

1280 butler: `lsst.daf.persistence.Butler` 

1281 

1282 Raises 

1283 ------ 

1284 RuntimeError 

1285 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 

1286 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 

1287 If cycleNumber > 0, then also checks for fgcmFitParameters, 

1288 fgcmFlaggedStars. 

1289 """ 

1290 

1291 if not butler.datasetExists('fgcmVisitCatalog'): 

1292 raise RuntimeError("Could not find fgcmVisitCatalog in repo!") 

1293 if not butler.datasetExists('fgcmStarObservations'): 

1294 raise RuntimeError("Could not find fgcmStarObservations in repo!") 

1295 if not butler.datasetExists('fgcmStarIds'): 

1296 raise RuntimeError("Could not find fgcmStarIds in repo!") 

1297 if not butler.datasetExists('fgcmStarIndices'): 

1298 raise RuntimeError("Could not find fgcmStarIndices in repo!") 

1299 if not butler.datasetExists('fgcmLookUpTable'): 

1300 raise RuntimeError("Could not find fgcmLookUpTable in repo!") 

1301 

1302 # Need additional datasets if we are not the initial cycle 

1303 if (self.config.cycleNumber > 0): 

1304 if not butler.datasetExists('fgcmFitParameters', 

1305 fgcmcycle=self.config.cycleNumber-1): 

1306 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" % 

1307 (self.config.cycleNumber-1)) 

1308 if not butler.datasetExists('fgcmFlaggedStars', 

1309 fgcmcycle=self.config.cycleNumber-1): 

1310 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" % 

1311 (self.config.cycleNumber-1)) 

1312 

1313 # And additional dataset if we want reference calibration 

1314 if self.config.doReferenceCalibration: 

1315 if not butler.datasetExists('fgcmReferenceStars'): 

1316 raise RuntimeError("Could not find fgcmReferenceStars in repo, and " 

1317 "doReferenceCalibration is True.") 

1318 

1319 def _loadParameters(self, parCat): 

1320 """ 

1321 Load FGCM parameters from a previous fit cycle 

1322 

1323 Parameters 

1324 ---------- 

1325 parCat : `lsst.afw.table.BaseCatalog` 

1326 Parameter catalog in afw table form. 

1327 

1328 Returns 

1329 ------- 

1330 inParInfo: `numpy.ndarray` 

1331 Numpy array parameter information formatted for input to fgcm 

1332 inParameters: `numpy.ndarray` 

1333 Numpy array parameter values formatted for input to fgcm 

1334 inSuperStar: `numpy.array` 

1335 Superstar flat formatted for input to fgcm 

1336 """ 

1337 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(',')) 

1338 parFitBands = np.array(parCat[0]['fitBands'].split(',')) 

1339 

1340 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'), 

1341 ('LUTFILTERNAMES', parLutFilterNames.dtype.str, 

1342 (parLutFilterNames.size, )), 

1343 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )), 

1344 ('LNTAUUNIT', 'f8'), 

1345 ('LNTAUSLOPEUNIT', 'f8'), 

1346 ('ALPHAUNIT', 'f8'), 

1347 ('LNPWVUNIT', 'f8'), 

1348 ('LNPWVSLOPEUNIT', 'f8'), 

1349 ('LNPWVQUADRATICUNIT', 'f8'), 

1350 ('LNPWVGLOBALUNIT', 'f8'), 

1351 ('O3UNIT', 'f8'), 

1352 ('QESYSUNIT', 'f8'), 

1353 ('FILTEROFFSETUNIT', 'f8'), 

1354 ('HASEXTERNALPWV', 'i2'), 

1355 ('HASEXTERNALTAU', 'i2')]) 

1356 inParInfo['NCCD'] = parCat['nCcd'] 

1357 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames 

1358 inParInfo['FITBANDS'][:] = parFitBands 

1359 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv'] 

1360 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau'] 

1361 

1362 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )), 

1363 ('PARO3', 'f8', (parCat['parO3'].size, )), 

1364 ('PARLNTAUINTERCEPT', 'f8', 

1365 (parCat['parLnTauIntercept'].size, )), 

1366 ('PARLNTAUSLOPE', 'f8', 

1367 (parCat['parLnTauSlope'].size, )), 

1368 ('PARLNPWVINTERCEPT', 'f8', 

1369 (parCat['parLnPwvIntercept'].size, )), 

1370 ('PARLNPWVSLOPE', 'f8', 

1371 (parCat['parLnPwvSlope'].size, )), 

1372 ('PARLNPWVQUADRATIC', 'f8', 

1373 (parCat['parLnPwvQuadratic'].size, )), 

1374 ('PARQESYSINTERCEPT', 'f8', 

1375 (parCat['parQeSysIntercept'].size, )), 

1376 ('COMPQESYSSLOPE', 'f8', 

1377 (parCat['compQeSysSlope'].size, )), 

1378 ('PARFILTEROFFSET', 'f8', 

1379 (parCat['parFilterOffset'].size, )), 

1380 ('PARFILTEROFFSETFITFLAG', 'i2', 

1381 (parCat['parFilterOffsetFitFlag'].size, )), 

1382 ('PARRETRIEVEDLNPWVSCALE', 'f8'), 

1383 ('PARRETRIEVEDLNPWVOFFSET', 'f8'), 

1384 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8', 

1385 (parCat['parRetrievedLnPwvNightlyOffset'].size, )), 

1386 ('COMPABSTHROUGHPUT', 'f8', 

1387 (parCat['compAbsThroughput'].size, )), 

1388 ('COMPREFOFFSET', 'f8', 

1389 (parCat['compRefOffset'].size, )), 

1390 ('COMPREFSIGMA', 'f8', 

1391 (parCat['compRefSigma'].size, )), 

1392 ('COMPMIRRORCHROMATICITY', 'f8', 

1393 (parCat['compMirrorChromaticity'].size, )), 

1394 ('MIRRORCHROMATICITYPIVOT', 'f8', 

1395 (parCat['mirrorChromaticityPivot'].size, )), 

1396 ('COMPMEDIANSEDSLOPE', 'f8', 

1397 (parCat['compMedianSedSlope'].size, )), 

1398 ('COMPAPERCORRPIVOT', 'f8', 

1399 (parCat['compAperCorrPivot'].size, )), 

1400 ('COMPAPERCORRSLOPE', 'f8', 

1401 (parCat['compAperCorrSlope'].size, )), 

1402 ('COMPAPERCORRSLOPEERR', 'f8', 

1403 (parCat['compAperCorrSlopeErr'].size, )), 

1404 ('COMPAPERCORRRANGE', 'f8', 

1405 (parCat['compAperCorrRange'].size, )), 

1406 ('COMPMODELERREXPTIMEPIVOT', 'f8', 

1407 (parCat['compModelErrExptimePivot'].size, )), 

1408 ('COMPMODELERRFWHMPIVOT', 'f8', 

1409 (parCat['compModelErrFwhmPivot'].size, )), 

1410 ('COMPMODELERRSKYPIVOT', 'f8', 

1411 (parCat['compModelErrSkyPivot'].size, )), 

1412 ('COMPMODELERRPARS', 'f8', 

1413 (parCat['compModelErrPars'].size, )), 

1414 ('COMPEXPGRAY', 'f8', 

1415 (parCat['compExpGray'].size, )), 

1416 ('COMPVARGRAY', 'f8', 

1417 (parCat['compVarGray'].size, )), 

1418 ('COMPEXPDELTAMAGBKG', 'f8', 

1419 (parCat['compExpDeltaMagBkg'].size, )), 

1420 ('COMPNGOODSTARPEREXP', 'i4', 

1421 (parCat['compNGoodStarPerExp'].size, )), 

1422 ('COMPSIGFGCM', 'f8', 

1423 (parCat['compSigFgcm'].size, )), 

1424 ('COMPSIGMACAL', 'f8', 

1425 (parCat['compSigmaCal'].size, )), 

1426 ('COMPRETRIEVEDLNPWV', 'f8', 

1427 (parCat['compRetrievedLnPwv'].size, )), 

1428 ('COMPRETRIEVEDLNPWVRAW', 'f8', 

1429 (parCat['compRetrievedLnPwvRaw'].size, )), 

1430 ('COMPRETRIEVEDLNPWVFLAG', 'i2', 

1431 (parCat['compRetrievedLnPwvFlag'].size, )), 

1432 ('COMPRETRIEVEDTAUNIGHT', 'f8', 

1433 (parCat['compRetrievedTauNight'].size, ))]) 

1434 

1435 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :] 

1436 inParams['PARO3'][:] = parCat['parO3'][0, :] 

1437 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :] 

1438 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :] 

1439 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :] 

1440 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :] 

1441 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :] 

1442 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :] 

1443 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :] 

1444 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :] 

1445 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :] 

1446 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale'] 

1447 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset'] 

1448 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :] 

1449 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :] 

1450 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :] 

1451 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :] 

1452 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :] 

1453 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :] 

1454 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :] 

1455 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :] 

1456 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :] 

1457 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :] 

1458 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :] 

1459 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :] 

1460 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :] 

1461 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :] 

1462 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :] 

1463 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :] 

1464 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :] 

1465 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :] 

1466 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :] 

1467 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :] 

1468 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :] 

1469 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :] 

1470 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :] 

1471 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :] 

1472 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :] 

1473 

1474 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8') 

1475 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape) 

1476 

1477 return (inParInfo, inParams, inSuperStar) 

1478 

1479 def _makeFgcmOutputDatasets(self, fgcmFitCycle): 

1480 """ 

1481 Persist FGCM datasets through the butler. 

1482 

1483 Parameters 

1484 ---------- 

1485 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 

1486 Fgcm Fit cycle object 

1487 """ 

1488 fgcmDatasetDict = {} 

1489 

1490 # Save the parameters 

1491 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays() 

1492 

1493 parSchema = afwTable.Schema() 

1494 

1495 comma = ',' 

1496 lutFilterNameString = comma.join([n.decode('utf-8') 

1497 for n in parInfo['LUTFILTERNAMES'][0]]) 

1498 fitBandString = comma.join([n.decode('utf-8') 

1499 for n in parInfo['FITBANDS'][0]]) 

1500 

1501 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1502 lutFilterNameString, fitBandString) 

1503 parCat = self._makeParCatalog(parSchema, parInfo, pars, 

1504 fgcmFitCycle.fgcmPars.parSuperStarFlat, 

1505 lutFilterNameString, fitBandString) 

1506 

1507 fgcmDatasetDict['fgcmFitParameters'] = parCat 

1508 

1509 # Save the indices of the flagged stars 

1510 # (stars that have been (a) reserved from the fit for testing and 

1511 # (b) bad stars that have failed quality checks.) 

1512 flagStarSchema = self._makeFlagStarSchema() 

1513 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices() 

1514 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct) 

1515 

1516 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat 

1517 

1518 # Save the zeropoint information and atmospheres only if desired 

1519 if self.outputZeropoints: 

1520 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1] 

1521 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1] 

1522 

1523 zptSchema = makeZptSchema(superStarChebSize, zptChebSize) 

1524 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct) 

1525 

1526 fgcmDatasetDict['fgcmZeropoints'] = zptCat 

1527 

1528 # Save atmosphere values 

1529 # These are generated by the same code that generates zeropoints 

1530 atmSchema = makeAtmSchema() 

1531 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct) 

1532 

1533 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat 

1534 

1535 # Save the standard stars (if configured) 

1536 if self.outputStandards: 

1537 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars) 

1538 stdSchema = makeStdSchema(len(goodBands)) 

1539 stdCat = makeStdCat(stdSchema, stdStruct, goodBands) 

1540 

1541 fgcmDatasetDict['fgcmStandardStars'] = stdCat 

1542 

1543 return fgcmDatasetDict 

1544 

1545 def _makeParSchema(self, parInfo, pars, parSuperStarFlat, 

1546 lutFilterNameString, fitBandString): 

1547 """ 

1548 Make the parameter persistence schema 

1549 

1550 Parameters 

1551 ---------- 

1552 parInfo: `numpy.ndarray` 

1553 Parameter information returned by fgcm 

1554 pars: `numpy.ndarray` 

1555 Parameter values returned by fgcm 

1556 parSuperStarFlat: `numpy.array` 

1557 Superstar flat values returned by fgcm 

1558 lutFilterNameString: `str` 

1559 Combined string of all the lutFilterNames 

1560 fitBandString: `str` 

1561 Combined string of all the fitBands 

1562 

1563 Returns 

1564 ------- 

1565 parSchema: `afwTable.schema` 

1566 """ 

1567 

1568 parSchema = afwTable.Schema() 

1569 

1570 # parameter info section 

1571 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') 

1572 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file', 

1573 size=len(lutFilterNameString)) 

1574 parSchema.addField('fitBands', type=str, doc='Bands that were fit', 

1575 size=len(fitBandString)) 

1576 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)') 

1577 parSchema.addField('lnTauSlopeUnit', type=np.float64, 

1578 doc='Step units for ln(AOD) slope') 

1579 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha') 

1580 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)') 

1581 parSchema.addField('lnPwvSlopeUnit', type=np.float64, 

1582 doc='Step units for ln(pwv) slope') 

1583 parSchema.addField('lnPwvQuadraticUnit', type=np.float64, 

1584 doc='Step units for ln(pwv) quadratic term') 

1585 parSchema.addField('lnPwvGlobalUnit', type=np.float64, 

1586 doc='Step units for global ln(pwv) parameters') 

1587 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3') 

1588 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray') 

1589 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset') 

1590 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv') 

1591 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau') 

1592 

1593 # parameter section 

1594 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector', 

1595 size=pars['PARALPHA'].size) 

1596 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector', 

1597 size=pars['PARO3'].size) 

1598 parSchema.addField('parLnTauIntercept', type='ArrayD', 

1599 doc='ln(Tau) intercept parameter vector', 

1600 size=pars['PARLNTAUINTERCEPT'].size) 

1601 parSchema.addField('parLnTauSlope', type='ArrayD', 

1602 doc='ln(Tau) slope parameter vector', 

1603 size=pars['PARLNTAUSLOPE'].size) 

1604 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector', 

1605 size=pars['PARLNPWVINTERCEPT'].size) 

1606 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector', 

1607 size=pars['PARLNPWVSLOPE'].size) 

1608 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector', 

1609 size=pars['PARLNPWVQUADRATIC'].size) 

1610 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector', 

1611 size=pars['PARQESYSINTERCEPT'].size) 

1612 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector', 

1613 size=pars[0]['COMPQESYSSLOPE'].size) 

1614 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector', 

1615 size=pars['PARFILTEROFFSET'].size) 

1616 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag', 

1617 size=pars['PARFILTEROFFSETFITFLAG'].size) 

1618 parSchema.addField('parRetrievedLnPwvScale', type=np.float64, 

1619 doc='Global scale for retrieved ln(pwv)') 

1620 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64, 

1621 doc='Global offset for retrieved ln(pwv)') 

1622 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD', 

1623 doc='Nightly offset for retrieved ln(pwv)', 

1624 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size) 

1625 parSchema.addField('compAbsThroughput', type='ArrayD', 

1626 doc='Absolute throughput (relative to transmission curves)', 

1627 size=pars['COMPABSTHROUGHPUT'].size) 

1628 parSchema.addField('compRefOffset', type='ArrayD', 

1629 doc='Offset between reference stars and calibrated stars', 

1630 size=pars['COMPREFOFFSET'].size) 

1631 parSchema.addField('compRefSigma', type='ArrayD', 

1632 doc='Width of reference star/calibrated star distribution', 

1633 size=pars['COMPREFSIGMA'].size) 

1634 parSchema.addField('compMirrorChromaticity', type='ArrayD', 

1635 doc='Computed mirror chromaticity terms', 

1636 size=pars['COMPMIRRORCHROMATICITY'].size) 

1637 parSchema.addField('mirrorChromaticityPivot', type='ArrayD', 

1638 doc='Mirror chromaticity pivot mjd', 

1639 size=pars['MIRRORCHROMATICITYPIVOT'].size) 

1640 parSchema.addField('compMedianSedSlope', type='ArrayD', 

1641 doc='Computed median SED slope (per band)', 

1642 size=pars['COMPMEDIANSEDSLOPE'].size) 

1643 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot', 

1644 size=pars['COMPAPERCORRPIVOT'].size) 

1645 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope', 

1646 size=pars['COMPAPERCORRSLOPE'].size) 

1647 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error', 

1648 size=pars['COMPAPERCORRSLOPEERR'].size) 

1649 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range', 

1650 size=pars['COMPAPERCORRRANGE'].size) 

1651 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot', 

1652 size=pars['COMPMODELERREXPTIMEPIVOT'].size) 

1653 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot', 

1654 size=pars['COMPMODELERRFWHMPIVOT'].size) 

1655 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot', 

1656 size=pars['COMPMODELERRSKYPIVOT'].size) 

1657 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters', 

1658 size=pars['COMPMODELERRPARS'].size) 

1659 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray', 

1660 size=pars['COMPEXPGRAY'].size) 

1661 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance', 

1662 size=pars['COMPVARGRAY'].size) 

1663 parSchema.addField('compExpDeltaMagBkg', type='ArrayD', 

1664 doc='Computed exposure offset due to background', 

1665 size=pars['COMPEXPDELTAMAGBKG'].size) 

1666 parSchema.addField('compNGoodStarPerExp', type='ArrayI', 

1667 doc='Computed number of good stars per exposure', 

1668 size=pars['COMPNGOODSTARPEREXP'].size) 

1669 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)', 

1670 size=pars['COMPSIGFGCM'].size) 

1671 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)', 

1672 size=pars['COMPSIGMACAL'].size) 

1673 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)', 

1674 size=pars['COMPRETRIEVEDLNPWV'].size) 

1675 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)', 

1676 size=pars['COMPRETRIEVEDLNPWVRAW'].size) 

1677 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag', 

1678 size=pars['COMPRETRIEVEDLNPWVFLAG'].size) 

1679 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)', 

1680 size=pars['COMPRETRIEVEDTAUNIGHT'].size) 

1681 # superstarflat section 

1682 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size', 

1683 size=4) 

1684 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)', 

1685 size=parSuperStarFlat.size) 

1686 

1687 return parSchema 

1688 

1689 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, 

1690 lutFilterNameString, fitBandString): 

1691 """ 

1692 Make the FGCM parameter catalog for persistence 

1693 

1694 Parameters 

1695 ---------- 

1696 parSchema: `lsst.afw.table.Schema` 

1697 Parameter catalog schema 

1698 pars: `numpy.ndarray` 

1699 FGCM parameters to put into parCat 

1700 parSuperStarFlat: `numpy.array` 

1701 FGCM superstar flat array to put into parCat 

1702 lutFilterNameString: `str` 

1703 Combined string of all the lutFilterNames 

1704 fitBandString: `str` 

1705 Combined string of all the fitBands 

1706 

1707 Returns 

1708 ------- 

1709 parCat: `afwTable.BasicCatalog` 

1710 Atmosphere and instrumental model parameter catalog for persistence 

1711 """ 

1712 

1713 parCat = afwTable.BaseCatalog(parSchema) 

1714 parCat.reserve(1) 

1715 

1716 # The parameter catalog just has one row, with many columns for all the 

1717 # atmosphere and instrument fit parameters 

1718 rec = parCat.addNew() 

1719 

1720 # info section 

1721 rec['nCcd'] = parInfo['NCCD'] 

1722 rec['lutFilterNames'] = lutFilterNameString 

1723 rec['fitBands'] = fitBandString 

1724 # note these are not currently supported here. 

1725 rec['hasExternalPwv'] = 0 

1726 rec['hasExternalTau'] = 0 

1727 

1728 # parameter section 

1729 

1730 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset'] 

1731 

1732 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope', 

1733 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic', 

1734 'parQeSysIntercept', 'compQeSysSlope', 

1735 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot', 

1736 'parFilterOffset', 'parFilterOffsetFitFlag', 

1737 'compAbsThroughput', 'compRefOffset', 'compRefSigma', 

1738 'compMirrorChromaticity', 'mirrorChromaticityPivot', 

1739 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange', 

1740 'compModelErrExptimePivot', 'compModelErrFwhmPivot', 

1741 'compModelErrSkyPivot', 'compModelErrPars', 

1742 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm', 

1743 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope', 

1744 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag', 

1745 'compRetrievedTauNight'] 

1746 

1747 for scalarName in scalarNames: 

1748 rec[scalarName] = pars[scalarName.upper()] 

1749 

1750 for arrName in arrNames: 

1751 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:] 

1752 

1753 # superstar section 

1754 rec['superstarSize'][:] = parSuperStarFlat.shape 

1755 rec['superstar'][:] = parSuperStarFlat.flatten() 

1756 

1757 return parCat 

1758 

1759 def _makeFlagStarSchema(self): 

1760 """ 

1761 Make the flagged-stars schema 

1762 

1763 Returns 

1764 ------- 

1765 flagStarSchema: `lsst.afw.table.Schema` 

1766 """ 

1767 

1768 flagStarSchema = afwTable.Schema() 

1769 

1770 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id') 

1771 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag') 

1772 

1773 return flagStarSchema 

1774 

1775 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct): 

1776 """ 

1777 Make the flagged star catalog for persistence 

1778 

1779 Parameters 

1780 ---------- 

1781 flagStarSchema: `lsst.afw.table.Schema` 

1782 Flagged star schema 

1783 flagStarStruct: `numpy.ndarray` 

1784 Flagged star structure from fgcm 

1785 

1786 Returns 

1787 ------- 

1788 flagStarCat: `lsst.afw.table.BaseCatalog` 

1789 Flagged star catalog for persistence 

1790 """ 

1791 

1792 flagStarCat = afwTable.BaseCatalog(flagStarSchema) 

1793 flagStarCat.resize(flagStarStruct.size) 

1794 

1795 flagStarCat['objId'][:] = flagStarStruct['OBJID'] 

1796 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG'] 

1797 

1798 return flagStarCat