Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44import lsst.afw.table as afwTable
46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47from .utilities import extractReferenceMags
48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50from .sedterms import SedboundarytermDict, SedtermDict
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57class FgcmFitCycleConfig(pexConfig.Config):
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc="Bands to run calibration",
62 dtype=str,
63 default=[],
64 )
65 fitFlag = pexConfig.ListField(
66 doc=("Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
70 dtype=int,
71 default=(0,),
72 optional=True,
73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
74 "It will be removed after v20. Use fitBands instead."),
75 )
76 fitBands = pexConfig.ListField(
77 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
78 "the atmosphere constrained from the 'fitBands' on the same night. "
79 "Must be a subset of `config.bands`"),
80 dtype=str,
81 default=[],
82 )
83 requiredFlag = pexConfig.ListField(
84 doc=("Flag for which bands are required for a star to be considered a calibration "
85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86 "be same length as config.bands, and matched band-by-band."),
87 dtype=int,
88 default=(0,),
89 optional=True,
90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
91 "It will be removed after v20. Use requiredBands instead."),
92 )
93 requiredBands = pexConfig.ListField(
94 doc=("Bands that are required for a star to be considered a calibration star. "
95 "Must be a subset of `config.bands`"),
96 dtype=str,
97 default=[],
98 )
99 filterMap = pexConfig.DictField(
100 doc="Mapping from 'filterName' to band.",
101 keytype=str,
102 itemtype=str,
103 default={},
104 )
105 doReferenceCalibration = pexConfig.Field(
106 doc="Use reference catalog as additional constraint on calibration",
107 dtype=bool,
108 default=True,
109 )
110 refStarSnMin = pexConfig.Field(
111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
112 dtype=float,
113 default=50.0,
114 )
115 refStarOutlierNSig = pexConfig.Field(
116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
118 dtype=float,
119 default=4.0,
120 )
121 applyRefStarColorCuts = pexConfig.Field(
122 doc="Apply color cuts to reference stars?",
123 dtype=bool,
124 default=True,
125 )
126 nCore = pexConfig.Field(
127 doc="Number of cores to use",
128 dtype=int,
129 default=4,
130 )
131 nStarPerRun = pexConfig.Field(
132 doc="Number of stars to run in each chunk",
133 dtype=int,
134 default=200000,
135 )
136 nExpPerRun = pexConfig.Field(
137 doc="Number of exposures to run in each chunk",
138 dtype=int,
139 default=1000,
140 )
141 reserveFraction = pexConfig.Field(
142 doc="Fraction of stars to reserve for testing",
143 dtype=float,
144 default=0.1,
145 )
146 freezeStdAtmosphere = pexConfig.Field(
147 doc="Freeze atmosphere parameters to standard (for testing)",
148 dtype=bool,
149 default=False,
150 )
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc="Precompute superstar flat for initial cycle",
153 dtype=bool,
154 default=False,
155 )
156 superStarSubCcd = pexConfig.Field(
157 doc="Compute superstar flat on sub-ccd scale",
158 dtype=bool,
159 default=True,
160 optional=True,
161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
162 "It will be removed after v20. Use superStarSubCcdDict instead."),
163 )
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166 "Must have one entry per band."),
167 keytype=str,
168 itemtype=bool,
169 default={},
170 )
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173 "Global default is first-order polynomials, and should be overridden "
174 "on a camera-by-camera basis depending on the ISR."),
175 dtype=int,
176 default=1,
177 )
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
180 "suppress high-order cross terms?"),
181 dtype=bool,
182 default=False,
183 )
184 superStarSigmaClip = pexConfig.Field(
185 doc="Number of sigma to clip outliers when selecting for superstar flats",
186 dtype=float,
187 default=5.0,
188 )
189 ccdGraySubCcd = pexConfig.Field(
190 doc="Compute CCD gray terms on sub-ccd scale",
191 dtype=bool,
192 default=False,
193 optional=True,
194 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
195 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
196 )
197 ccdGraySubCcdDict = pexConfig.DictField(
198 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
199 "('ccd gray') on a sub-ccd scale."),
200 keytype=str,
201 itemtype=bool,
202 default={},
203 )
204 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
206 dtype=int,
207 default=1,
208 )
209 ccdGraySubCcdTriangular = pexConfig.Field(
210 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
211 "suppress high-order cross terms?"),
212 dtype=bool,
213 default=True,
214 )
215 cycleNumber = pexConfig.Field(
216 doc=("FGCM fit cycle number. This is automatically incremented after each run "
217 "and stage of outlier rejection. See cookbook for details."),
218 dtype=int,
219 default=None,
220 )
221 isFinalCycle = pexConfig.Field(
222 doc=("Is this the final cycle of the fitting? Will automatically compute final "
223 "selection of stars and photometric exposures, and will output zeropoints "
224 "and standard stars for use in fgcmOutputProducts"),
225 dtype=bool,
226 default=False,
227 )
228 maxIterBeforeFinalCycle = pexConfig.Field(
229 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
230 "will always be 0 in the final cycle for cleanup and final selection."),
231 dtype=int,
232 default=50,
233 )
234 utBoundary = pexConfig.Field(
235 doc="Boundary (in UTC) from day-to-day",
236 dtype=float,
237 default=None,
238 )
239 washMjds = pexConfig.ListField(
240 doc="Mirror wash MJDs",
241 dtype=float,
242 default=(0.0,),
243 )
244 epochMjds = pexConfig.ListField(
245 doc="Epoch boundaries in MJD",
246 dtype=float,
247 default=(0.0,),
248 )
249 minObsPerBand = pexConfig.Field(
250 doc="Minimum good observations per band",
251 dtype=int,
252 default=2,
253 )
254 # TODO: When DM-16511 is done, it will be possible to get the
255 # telescope latitude directly from the camera.
256 latitude = pexConfig.Field(
257 doc="Observatory latitude",
258 dtype=float,
259 default=None,
260 )
261 brightObsGrayMax = pexConfig.Field(
262 doc="Maximum gray extinction to be considered bright observation",
263 dtype=float,
264 default=0.15,
265 )
266 minStarPerCcd = pexConfig.Field(
267 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
268 "CCDs with fewer stars will have their calibration estimated from other "
269 "CCDs in the same visit, with zeropoint error increased accordingly."),
270 dtype=int,
271 default=5,
272 )
273 minCcdPerExp = pexConfig.Field(
274 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
275 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
276 dtype=int,
277 default=5,
278 )
279 maxCcdGrayErr = pexConfig.Field(
280 doc="Maximum error on CCD gray offset to be considered photometric",
281 dtype=float,
282 default=0.05,
283 )
284 minStarPerExp = pexConfig.Field(
285 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
286 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
287 dtype=int,
288 default=600,
289 )
290 minExpPerNight = pexConfig.Field(
291 doc="Minimum number of good exposures/visits to consider a partly photometric night",
292 dtype=int,
293 default=10,
294 )
295 expGrayInitialCut = pexConfig.Field(
296 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
297 "observations."),
298 dtype=float,
299 default=-0.25,
300 )
301 expGrayPhotometricCut = pexConfig.ListField(
302 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
303 "Must be same length as config.bands, and matched band-by-band."),
304 dtype=float,
305 default=(0.0,),
306 optional=True,
307 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
308 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
309 )
310 expGrayPhotometricCutDict = pexConfig.DictField(
311 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
312 "('gray term') for a visit to be considered photometric. Must have one "
313 "entry per band. Broad-band filters should be -0.05."),
314 keytype=str,
315 itemtype=float,
316 default={},
317 )
318 expGrayHighCut = pexConfig.ListField(
319 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
320 "Must be same length as config.bands, and matched band-by-band."),
321 dtype=float,
322 default=(0.0,),
323 optional=True,
324 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
325 "It will be removed after v20. Use expGrayHighCutDict instead."),
326 )
327 expGrayHighCutDict = pexConfig.DictField(
328 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
329 "('gray term') for a visit to be considered photometric. Must have one "
330 "entry per band. Broad-band filters should be 0.2."),
331 keytype=str,
332 itemtype=float,
333 default={},
334 )
335 expGrayRecoverCut = pexConfig.Field(
336 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
337 "Visits with more gray extinction will only get CCD zeropoints if there are "
338 "sufficient star observations (minStarPerCcd) on that CCD."),
339 dtype=float,
340 default=-1.0,
341 )
342 expVarGrayPhotometricCut = pexConfig.Field(
343 doc="Maximum exposure variance to be considered possibly photometric",
344 dtype=float,
345 default=0.0005,
346 optional=True,
347 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
348 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
349 )
350 expVarGrayPhotometricCutDict = pexConfig.DictField(
351 doc=("Per-band specification on maximum exposure variance to be considered possibly "
352 "photometric. Must have one entry per band. Broad-band filters should be "
353 "0.0005."),
354 keytype=str,
355 itemtype=float,
356 default={},
357 )
358 expGrayErrRecoverCut = pexConfig.Field(
359 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
360 "Visits with more gray variance will only get CCD zeropoints if there are "
361 "sufficient star observations (minStarPerCcd) on that CCD."),
362 dtype=float,
363 default=0.05,
364 )
365 aperCorrFitNBins = pexConfig.Field(
366 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
367 "no fit will be performed, and the config.aperCorrInputSlopes will be "
368 "used if available."),
369 dtype=int,
370 default=10,
371 )
372 aperCorrInputSlopes = pexConfig.ListField(
373 doc=("Aperture correction input slope parameters. These are used on the first "
374 "fit iteration, and aperture correction parameters will be updated from "
375 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
376 "when there is insufficient data to fit the parameters (e.g. tract mode). "
377 "If set, must be same length as config.bands, and matched band-by-band."),
378 dtype=float,
379 default=[],
380 optional=True,
381 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
382 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
383 )
384 aperCorrInputSlopeDict = pexConfig.DictField(
385 doc=("Per-band specification of aperture correction input slope parameters. These "
386 "are used on the first fit iteration, and aperture correction parameters will "
387 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
388 "to set this when there is insufficient data to fit the parameters (e.g. "
389 "tract mode)."),
390 keytype=str,
391 itemtype=float,
392 default={},
393 )
394 sedFudgeFactors = pexConfig.ListField(
395 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
396 "config.bands, and matched band-by-band."),
397 dtype=float,
398 default=(0,),
399 optional=True,
400 deprecated=("This field has been deprecated and will be removed after v20. "
401 "Please use sedSlopeTermMap and sedSlopeMap."),
402 )
403 sedboundaryterms = pexConfig.ConfigField(
404 doc="Mapping from bands to SED boundary term names used is sedterms.",
405 dtype=SedboundarytermDict,
406 )
407 sedterms = pexConfig.ConfigField(
408 doc="Mapping from terms to bands for fgcm linear SED approximations.",
409 dtype=SedtermDict,
410 )
411 sigFgcmMaxErr = pexConfig.Field(
412 doc="Maximum mag error for fitting sigma_FGCM",
413 dtype=float,
414 default=0.01,
415 )
416 sigFgcmMaxEGray = pexConfig.ListField(
417 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
418 "May be 1 element (same for all bands) or the same length as config.bands."),
419 dtype=float,
420 default=(0.05,),
421 optional=True,
422 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
423 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
424 )
425 sigFgcmMaxEGrayDict = pexConfig.DictField(
426 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
427 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
428 "should be 0.05."),
429 keytype=str,
430 itemtype=float,
431 default={},
432 )
433 ccdGrayMaxStarErr = pexConfig.Field(
434 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
435 "computation"),
436 dtype=float,
437 default=0.10,
438 )
439 approxThroughput = pexConfig.ListField(
440 doc=("Approximate overall throughput at start of calibration observations. "
441 "May be 1 element (same for all bands) or the same length as config.bands."),
442 dtype=float,
443 default=(1.0, ),
444 optional=True,
445 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
446 "It will be removed after v20. Use approxThroughputDict instead."),
447 )
448 approxThroughputDict = pexConfig.DictField(
449 doc=("Per-band specification of the approximate overall throughput at the start of "
450 "calibration observations. Must have one entry per band. Typically should "
451 "be 1.0."),
452 keytype=str,
453 itemtype=float,
454 default={},
455 )
456 sigmaCalRange = pexConfig.ListField(
457 doc="Allowed range for systematic error floor estimation",
458 dtype=float,
459 default=(0.001, 0.003),
460 )
461 sigmaCalFitPercentile = pexConfig.ListField(
462 doc="Magnitude percentile range to fit systematic error floor",
463 dtype=float,
464 default=(0.05, 0.15),
465 )
466 sigmaCalPlotPercentile = pexConfig.ListField(
467 doc="Magnitude percentile range to plot systematic error floor",
468 dtype=float,
469 default=(0.05, 0.95),
470 )
471 sigma0Phot = pexConfig.Field(
472 doc="Systematic error floor for all zeropoints",
473 dtype=float,
474 default=0.003,
475 )
476 mapLongitudeRef = pexConfig.Field(
477 doc="Reference longitude for plotting maps",
478 dtype=float,
479 default=0.0,
480 )
481 mapNSide = pexConfig.Field(
482 doc="Healpix nside for plotting maps",
483 dtype=int,
484 default=256,
485 )
486 outfileBase = pexConfig.Field(
487 doc="Filename start for plot output files",
488 dtype=str,
489 default=None,
490 )
491 starColorCuts = pexConfig.ListField(
492 doc="Encoded star-color cuts (to be cleaned up)",
493 dtype=str,
494 default=("NO_DATA",),
495 )
496 colorSplitIndices = pexConfig.ListField(
497 doc="Band indices to use to split stars by color",
498 dtype=int,
499 default=None,
500 optional=True,
501 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
502 "It will be removed after v20. Use colorSplitBands instead."),
503 )
504 colorSplitBands = pexConfig.ListField(
505 doc="Band names to use to split stars by color. Must have 2 entries.",
506 dtype=str,
507 length=2,
508 default=('g', 'i'),
509 )
510 modelMagErrors = pexConfig.Field(
511 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
512 dtype=bool,
513 default=True,
514 )
515 useQuadraticPwv = pexConfig.Field(
516 doc="Model PWV with a quadratic term for variation through the night?",
517 dtype=bool,
518 default=False,
519 )
520 instrumentParsPerBand = pexConfig.Field(
521 doc=("Model instrumental parameters per band? "
522 "Otherwise, instrumental parameters (QE changes with time) are "
523 "shared among all bands."),
524 dtype=bool,
525 default=False,
526 )
527 instrumentSlopeMinDeltaT = pexConfig.Field(
528 doc=("Minimum time change (in days) between observations to use in constraining "
529 "instrument slope."),
530 dtype=float,
531 default=20.0,
532 )
533 fitMirrorChromaticity = pexConfig.Field(
534 doc="Fit (intraband) mirror chromatic term?",
535 dtype=bool,
536 default=False,
537 )
538 coatingMjds = pexConfig.ListField(
539 doc="Mirror coating dates in MJD",
540 dtype=float,
541 default=(0.0,),
542 )
543 outputStandardsBeforeFinalCycle = pexConfig.Field(
544 doc="Output standard stars prior to final cycle? Used in debugging.",
545 dtype=bool,
546 default=False,
547 )
548 outputZeropointsBeforeFinalCycle = pexConfig.Field(
549 doc="Output standard stars prior to final cycle? Used in debugging.",
550 dtype=bool,
551 default=False,
552 )
553 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
554 doc=("Use star repeatability (instead of exposures) for computing photometric "
555 "cuts? Recommended for tract mode or bands with few exposures. "
556 "May be 1 element (same for all bands) or the same length as config.bands."),
557 dtype=bool,
558 default=(False,),
559 optional=True,
560 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
561 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
562 )
563 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
564 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
565 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
566 keytype=str,
567 itemtype=bool,
568 default={},
569 )
570 autoPhotometricCutNSig = pexConfig.Field(
571 doc=("Number of sigma for automatic computation of (low) photometric cut. "
572 "Cut is based on exposure gray width (per band), unless "
573 "useRepeatabilityForExpGrayCuts is set, in which case the star "
574 "repeatability is used (also per band)."),
575 dtype=float,
576 default=3.0,
577 )
578 autoHighCutNSig = pexConfig.Field(
579 doc=("Number of sigma for automatic computation of (high) outlier cut. "
580 "Cut is based on exposure gray width (per band), unless "
581 "useRepeatabilityForExpGrayCuts is set, in which case the star "
582 "repeatability is used (also per band)."),
583 dtype=float,
584 default=4.0,
585 )
586 quietMode = pexConfig.Field(
587 doc="Be less verbose with logging.",
588 dtype=bool,
589 default=False,
590 )
592 def setDefaults(self):
593 pass
595 def validate(self):
596 super().validate()
598 for band in self.fitBands:
599 if band not in self.bands:
600 msg = 'fitBand %s not in bands' % (band)
601 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
602 for band in self.requiredBands:
603 if band not in self.bands:
604 msg = 'requiredBand %s not in bands' % (band)
605 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
606 for band in self.colorSplitBands:
607 if band not in self.bands:
608 msg = 'colorSplitBand %s not in bands' % (band)
609 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
610 for band in self.bands:
611 if band not in self.superStarSubCcdDict:
612 msg = 'band %s not in superStarSubCcdDict' % (band)
613 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
614 self, msg)
615 if band not in self.ccdGraySubCcdDict:
616 msg = 'band %s not in ccdGraySubCcdDict' % (band)
617 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
618 self, msg)
619 if band not in self.expGrayPhotometricCutDict:
620 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
621 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
622 self, msg)
623 if band not in self.expGrayHighCutDict:
624 msg = 'band %s not in expGrayHighCutDict' % (band)
625 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
626 self, msg)
627 if band not in self.expVarGrayPhotometricCutDict:
628 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
629 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
630 self, msg)
631 if band not in self.sigFgcmMaxEGrayDict:
632 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
633 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
634 self, msg)
635 if band not in self.approxThroughputDict:
636 msg = 'band %s not in approxThroughputDict' % (band)
637 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
638 self, msg)
639 if band not in self.useRepeatabilityForExpGrayCutsDict:
640 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
641 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
642 self, msg)
645class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
646 """Subclass of TaskRunner for fgcmFitCycleTask
648 fgcmFitCycleTask.run() takes one argument, the butler, and uses
649 stars and visits previously extracted from dataRefs by
650 fgcmBuildStars.
651 This Runner does not perform any dataRef parallelization, but the FGCM
652 code called by the Task uses python multiprocessing (see the "ncores"
653 config option).
654 """
656 @staticmethod
657 def getTargetList(parsedCmd):
658 """
659 Return a list with one element, the butler.
660 """
661 return [parsedCmd.butler]
663 def __call__(self, butler):
664 """
665 Parameters
666 ----------
667 butler: `lsst.daf.persistence.Butler`
669 Returns
670 -------
671 exitStatus: `list` with `pipeBase.Struct`
672 exitStatus (0: success; 1: failure)
673 """
675 task = self.TaskClass(config=self.config, log=self.log)
677 exitStatus = 0
678 if self.doRaise:
679 task.runDataRef(butler)
680 else:
681 try:
682 task.runDataRef(butler)
683 except Exception as e:
684 exitStatus = 1
685 task.log.fatal("Failed: %s" % e)
686 if not isinstance(e, pipeBase.TaskError):
687 traceback.print_exc(file=sys.stderr)
689 task.writeMetadata(butler)
691 # The task does not return any results:
692 return [pipeBase.Struct(exitStatus=exitStatus)]
694 def run(self, parsedCmd):
695 """
696 Run the task, with no multiprocessing
698 Parameters
699 ----------
700 parsedCmd: ArgumentParser parsed command line
701 """
703 resultList = []
705 if self.precall(parsedCmd):
706 targetList = self.getTargetList(parsedCmd)
707 # make sure that we only get 1
708 resultList = self(targetList[0])
710 return resultList
713class FgcmFitCycleTask(pipeBase.CmdLineTask):
714 """
715 Run Single fit cycle for FGCM global calibration
716 """
718 ConfigClass = FgcmFitCycleConfig
719 RunnerClass = FgcmFitCycleRunner
720 _DefaultName = "fgcmFitCycle"
722 def __init__(self, butler=None, **kwargs):
723 """
724 Instantiate an fgcmFitCycle.
726 Parameters
727 ----------
728 butler : `lsst.daf.persistence.Butler`
729 """
731 pipeBase.CmdLineTask.__init__(self, **kwargs)
733 # no saving of metadata for now
734 def _getMetadataName(self):
735 return None
737 @pipeBase.timeMethod
738 def runDataRef(self, butler):
739 """
740 Run a single fit cycle for FGCM
742 Parameters
743 ----------
744 butler: `lsst.daf.persistence.Butler`
745 """
747 self._fgcmFitCycle(butler)
749 def writeConfig(self, butler, clobber=False, doBackup=True):
750 """Write the configuration used for processing the data, or check that an existing
751 one is equal to the new one if present. This is an override of the regular
752 version from pipe_base that knows about fgcmcycle.
754 Parameters
755 ----------
756 butler : `lsst.daf.persistence.Butler`
757 Data butler used to write the config. The config is written to dataset type
758 `CmdLineTask._getConfigName`.
759 clobber : `bool`, optional
760 A boolean flag that controls what happens if a config already has been saved:
761 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
762 - `False`: raise `TaskError` if this config does not match the existing config.
763 doBackup : `bool`, optional
764 Set to `True` to backup the config files if clobbering.
765 """
766 configName = self._getConfigName()
767 if configName is None:
768 return
769 if clobber:
770 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
771 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
772 # this may be subject to a race condition; see #2789
773 try:
774 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
775 except Exception as exc:
776 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
777 (configName, exc))
779 def logConfigMismatch(msg):
780 self.log.fatal("Comparing configuration: %s", msg)
782 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
783 raise pipeBase.TaskError(
784 ("Config does not match existing task config %r on disk; tasks configurations " +
785 "must be consistent within the same output repo (override with --clobber-config)") %
786 (configName,))
787 else:
788 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
790 def _fgcmFitCycle(self, butler):
791 """
792 Run the fit cycle
794 Parameters
795 ----------
796 butler: `lsst.daf.persistence.Butler`
797 """
799 self._checkDatasetsExist(butler)
801 # Set defaults on whether to output standards and zeropoints
802 self.maxIter = self.config.maxIterBeforeFinalCycle
803 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
804 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
805 self.resetFitParameters = True
807 if self.config.isFinalCycle:
808 # This is the final fit cycle, so we do not want to reset fit
809 # parameters, we want to run a final "clean-up" with 0 fit iterations,
810 # and we always want to output standards and zeropoints
811 self.maxIter = 0
812 self.outputStandards = True
813 self.outputZeropoints = True
814 self.resetFitParameters = False
816 camera = butler.get('camera')
817 configDict = makeConfigDict(self.config, self.log, camera,
818 self.maxIter, self.resetFitParameters,
819 self.outputZeropoints)
821 lutCat = butler.get('fgcmLookUpTable')
822 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
823 del lutCat
825 # next we need the exposure/visit information
827 # fgcmExpInfo = self._loadVisitCatalog(butler)
828 visitCat = butler.get('fgcmVisitCatalog')
829 fgcmExpInfo = translateVisitCatalog(visitCat)
830 del visitCat
832 # Use the first orientation.
833 # TODO: DM-21215 will generalize to arbitrary camera orientations
834 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
836 noFitsDict = {'lutIndex': lutIndexVals,
837 'lutStd': lutStd,
838 'expInfo': fgcmExpInfo,
839 'ccdOffsets': ccdOffsets}
841 # set up the fitter object
842 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
843 noFitsDict=noFitsDict, noOutput=True)
845 # create the parameter object
846 if (fgcmFitCycle.initialCycle):
847 # cycle = 0, initial cycle
848 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
849 fgcmLut,
850 fgcmExpInfo)
851 else:
852 inParInfo, inParams, inSuperStar = self._loadParameters(butler)
853 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
854 fgcmExpInfo,
855 inParInfo,
856 inParams,
857 inSuperStar)
859 lastCycle = configDict['cycleNumber'] - 1
861 # set up the stars...
862 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
864 starObs = butler.get('fgcmStarObservations')
865 starIds = butler.get('fgcmStarIds')
866 starIndices = butler.get('fgcmStarIndices')
868 # grab the flagged stars if available
869 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
870 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
871 flagId = flaggedStars['objId'][:]
872 flagFlag = flaggedStars['objFlag'][:]
873 else:
874 flaggedStars = None
875 flagId = None
876 flagFlag = None
878 if self.config.doReferenceCalibration:
879 refStars = butler.get('fgcmReferenceStars')
881 refMag, refMagErr = extractReferenceMags(refStars,
882 self.config.bands,
883 self.config.filterMap)
884 refId = refStars['fgcm_id'][:]
885 else:
886 refStars = None
887 refId = None
888 refMag = None
889 refMagErr = None
891 # match star observations to visits
892 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
893 # actually be transferred into fgcm using the indexing below.
894 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
896 # The fgcmStars.loadStars method will copy all the star information into
897 # special shared memory objects that will not blow up the memory usage when
898 # used with python multiprocessing. Once all the numbers are copied,
899 # it is necessary to release all references to the objects that previously
900 # stored the data to ensure that the garbage collector can clear the memory,
901 # and ensure that this memory is not copied when multiprocessing kicks in.
903 # We determine the conversion from the native units (typically radians) to
904 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
905 # numpy arrays rather than Angles, which would we approximately 600x slower.
906 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
908 fgcmStars.loadStars(fgcmPars,
909 starObs['visit'][starIndices['obsIndex']],
910 starObs['ccd'][starIndices['obsIndex']],
911 starObs['ra'][starIndices['obsIndex']] * conv,
912 starObs['dec'][starIndices['obsIndex']] * conv,
913 starObs['instMag'][starIndices['obsIndex']],
914 starObs['instMagErr'][starIndices['obsIndex']],
915 fgcmExpInfo['FILTERNAME'][visitIndex],
916 starIds['fgcm_id'][:],
917 starIds['ra'][:],
918 starIds['dec'][:],
919 starIds['obsArrIndex'][:],
920 starIds['nObs'][:],
921 obsX=starObs['x'][starIndices['obsIndex']],
922 obsY=starObs['y'][starIndices['obsIndex']],
923 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
924 refID=refId,
925 refMag=refMag,
926 refMagErr=refMagErr,
927 flagID=flagId,
928 flagFlag=flagFlag,
929 computeNobs=True)
931 # Release all references to temporary objects holding star data (see above)
932 del starObs
933 del starIds
934 del starIndices
935 del flagId
936 del flagFlag
937 del flaggedStars
938 del refStars
939 del refId
940 del refMag
941 del refMagErr
943 # and set the bits in the cycle object
944 fgcmFitCycle.setLUT(fgcmLut)
945 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
946 fgcmFitCycle.setPars(fgcmPars)
948 # finish the setup
949 fgcmFitCycle.finishSetup()
951 # and run
952 fgcmFitCycle.run()
954 ##################
955 # Persistance
956 ##################
958 self._persistFgcmDatasets(butler, fgcmFitCycle)
960 # Output the config for the next cycle
961 # We need to make a copy since the input one has been frozen
963 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
964 i, b in enumerate(self.config.bands)}
965 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
966 i, band in enumerate(self.config.bands)}
968 outConfig = copy.copy(self.config)
969 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
970 precomputeSuperStarInitialCycle=False,
971 freezeStdAtmosphere=False,
972 expGrayPhotometricCutDict=updatedPhotometricCutDict,
973 expGrayHighCutDict=updatedHighCutDict)
974 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
975 outConfig.cycleNumber)
976 outConfig.save(configFileName)
978 if self.config.isFinalCycle == 1:
979 # We are done, ready to output products
980 self.log.info("Everything is in place to run fgcmOutputProducts.py")
981 else:
982 self.log.info("Saved config for next cycle to %s" % (configFileName))
983 self.log.info("Be sure to look at:")
984 self.log.info(" config.expGrayPhotometricCut")
985 self.log.info(" config.expGrayHighCut")
986 self.log.info("If you are satisfied with the fit, please set:")
987 self.log.info(" config.isFinalCycle = True")
989 def _checkDatasetsExist(self, butler):
990 """
991 Check if necessary datasets exist to run fgcmFitCycle
993 Parameters
994 ----------
995 butler: `lsst.daf.persistence.Butler`
997 Raises
998 ------
999 RuntimeError
1000 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1001 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1002 If cycleNumber > 0, then also checks for fgcmFitParameters,
1003 fgcmFlaggedStars.
1004 """
1006 if not butler.datasetExists('fgcmVisitCatalog'):
1007 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1008 if not butler.datasetExists('fgcmStarObservations'):
1009 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1010 if not butler.datasetExists('fgcmStarIds'):
1011 raise RuntimeError("Could not find fgcmStarIds in repo!")
1012 if not butler.datasetExists('fgcmStarIndices'):
1013 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1014 if not butler.datasetExists('fgcmLookUpTable'):
1015 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1017 # Need additional datasets if we are not the initial cycle
1018 if (self.config.cycleNumber > 0):
1019 if not butler.datasetExists('fgcmFitParameters',
1020 fgcmcycle=self.config.cycleNumber-1):
1021 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1022 (self.config.cycleNumber-1))
1023 if not butler.datasetExists('fgcmFlaggedStars',
1024 fgcmcycle=self.config.cycleNumber-1):
1025 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1026 (self.config.cycleNumber-1))
1028 # And additional dataset if we want reference calibration
1029 if self.config.doReferenceCalibration:
1030 if not butler.datasetExists('fgcmReferenceStars'):
1031 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1032 "doReferenceCalibration is True.")
1034 def _loadParameters(self, butler):
1035 """
1036 Load FGCM parameters from a previous fit cycle
1038 Parameters
1039 ----------
1040 butler: `lsst.daf.persistence.Butler`
1042 Returns
1043 -------
1044 inParInfo: `numpy.ndarray`
1045 Numpy array parameter information formatted for input to fgcm
1046 inParameters: `numpy.ndarray`
1047 Numpy array parameter values formatted for input to fgcm
1048 inSuperStar: `numpy.array`
1049 Superstar flat formatted for input to fgcm
1050 """
1052 # note that we already checked that this is available
1053 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1055 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1056 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1058 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1059 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1060 (parLutFilterNames.size, )),
1061 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1062 ('LNTAUUNIT', 'f8'),
1063 ('LNTAUSLOPEUNIT', 'f8'),
1064 ('ALPHAUNIT', 'f8'),
1065 ('LNPWVUNIT', 'f8'),
1066 ('LNPWVSLOPEUNIT', 'f8'),
1067 ('LNPWVQUADRATICUNIT', 'f8'),
1068 ('LNPWVGLOBALUNIT', 'f8'),
1069 ('O3UNIT', 'f8'),
1070 ('QESYSUNIT', 'f8'),
1071 ('FILTEROFFSETUNIT', 'f8'),
1072 ('HASEXTERNALPWV', 'i2'),
1073 ('HASEXTERNALTAU', 'i2')])
1074 inParInfo['NCCD'] = parCat['nCcd']
1075 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1076 inParInfo['FITBANDS'][:] = parFitBands
1077 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1078 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1080 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1081 ('PARO3', 'f8', (parCat['parO3'].size, )),
1082 ('PARLNTAUINTERCEPT', 'f8',
1083 (parCat['parLnTauIntercept'].size, )),
1084 ('PARLNTAUSLOPE', 'f8',
1085 (parCat['parLnTauSlope'].size, )),
1086 ('PARLNPWVINTERCEPT', 'f8',
1087 (parCat['parLnPwvIntercept'].size, )),
1088 ('PARLNPWVSLOPE', 'f8',
1089 (parCat['parLnPwvSlope'].size, )),
1090 ('PARLNPWVQUADRATIC', 'f8',
1091 (parCat['parLnPwvQuadratic'].size, )),
1092 ('PARQESYSINTERCEPT', 'f8',
1093 (parCat['parQeSysIntercept'].size, )),
1094 ('COMPQESYSSLOPE', 'f8',
1095 (parCat['compQeSysSlope'].size, )),
1096 ('PARFILTEROFFSET', 'f8',
1097 (parCat['parFilterOffset'].size, )),
1098 ('PARFILTEROFFSETFITFLAG', 'i2',
1099 (parCat['parFilterOffsetFitFlag'].size, )),
1100 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1101 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1102 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1103 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1104 ('COMPABSTHROUGHPUT', 'f8',
1105 (parCat['compAbsThroughput'].size, )),
1106 ('COMPREFOFFSET', 'f8',
1107 (parCat['compRefOffset'].size, )),
1108 ('COMPREFSIGMA', 'f8',
1109 (parCat['compRefSigma'].size, )),
1110 ('COMPMIRRORCHROMATICITY', 'f8',
1111 (parCat['compMirrorChromaticity'].size, )),
1112 ('MIRRORCHROMATICITYPIVOT', 'f8',
1113 (parCat['mirrorChromaticityPivot'].size, )),
1114 ('COMPAPERCORRPIVOT', 'f8',
1115 (parCat['compAperCorrPivot'].size, )),
1116 ('COMPAPERCORRSLOPE', 'f8',
1117 (parCat['compAperCorrSlope'].size, )),
1118 ('COMPAPERCORRSLOPEERR', 'f8',
1119 (parCat['compAperCorrSlopeErr'].size, )),
1120 ('COMPAPERCORRRANGE', 'f8',
1121 (parCat['compAperCorrRange'].size, )),
1122 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1123 (parCat['compModelErrExptimePivot'].size, )),
1124 ('COMPMODELERRFWHMPIVOT', 'f8',
1125 (parCat['compModelErrFwhmPivot'].size, )),
1126 ('COMPMODELERRSKYPIVOT', 'f8',
1127 (parCat['compModelErrSkyPivot'].size, )),
1128 ('COMPMODELERRPARS', 'f8',
1129 (parCat['compModelErrPars'].size, )),
1130 ('COMPEXPGRAY', 'f8',
1131 (parCat['compExpGray'].size, )),
1132 ('COMPVARGRAY', 'f8',
1133 (parCat['compVarGray'].size, )),
1134 ('COMPNGOODSTARPEREXP', 'i4',
1135 (parCat['compNGoodStarPerExp'].size, )),
1136 ('COMPSIGFGCM', 'f8',
1137 (parCat['compSigFgcm'].size, )),
1138 ('COMPSIGMACAL', 'f8',
1139 (parCat['compSigmaCal'].size, )),
1140 ('COMPRETRIEVEDLNPWV', 'f8',
1141 (parCat['compRetrievedLnPwv'].size, )),
1142 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1143 (parCat['compRetrievedLnPwvRaw'].size, )),
1144 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1145 (parCat['compRetrievedLnPwvFlag'].size, )),
1146 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1147 (parCat['compRetrievedTauNight'].size, ))])
1149 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1150 inParams['PARO3'][:] = parCat['parO3'][0, :]
1151 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1152 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1153 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1154 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1155 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1156 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1157 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1158 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1159 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1160 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1161 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1162 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1163 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1164 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1165 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1166 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1167 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1168 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1169 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1170 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1171 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1172 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1173 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1174 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1175 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1176 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1177 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1178 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1179 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1180 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1181 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1182 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1183 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1184 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1186 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1187 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1189 return (inParInfo, inParams, inSuperStar)
1191 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1192 """
1193 Persist FGCM datasets through the butler.
1195 Parameters
1196 ----------
1197 butler: `lsst.daf.persistence.Butler`
1198 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1199 Fgcm Fit cycle object
1200 """
1202 # Save the parameters
1203 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1205 parSchema = afwTable.Schema()
1207 comma = ','
1208 lutFilterNameString = comma.join([n.decode('utf-8')
1209 for n in parInfo['LUTFILTERNAMES'][0]])
1210 fitBandString = comma.join([n.decode('utf-8')
1211 for n in parInfo['FITBANDS'][0]])
1213 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1214 lutFilterNameString, fitBandString)
1215 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1216 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1217 lutFilterNameString, fitBandString)
1219 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1221 # Save the indices of the flagged stars
1222 # (stars that have been (a) reserved from the fit for testing and
1223 # (b) bad stars that have failed quality checks.)
1224 flagStarSchema = self._makeFlagStarSchema()
1225 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1226 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1228 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1230 # Save the zeropoint information and atmospheres only if desired
1231 if self.outputZeropoints:
1232 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1233 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1235 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1236 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1238 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1240 # Save atmosphere values
1241 # These are generated by the same code that generates zeropoints
1242 atmSchema = makeAtmSchema()
1243 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1245 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1247 # Save the standard stars (if configured)
1248 if self.outputStandards:
1249 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1250 stdSchema = makeStdSchema(len(goodBands))
1251 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1253 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1255 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1256 lutFilterNameString, fitBandString):
1257 """
1258 Make the parameter persistence schema
1260 Parameters
1261 ----------
1262 parInfo: `numpy.ndarray`
1263 Parameter information returned by fgcm
1264 pars: `numpy.ndarray`
1265 Parameter values returned by fgcm
1266 parSuperStarFlat: `numpy.array`
1267 Superstar flat values returned by fgcm
1268 lutFilterNameString: `str`
1269 Combined string of all the lutFilterNames
1270 fitBandString: `str`
1271 Combined string of all the fitBands
1273 Returns
1274 -------
1275 parSchema: `afwTable.schema`
1276 """
1278 parSchema = afwTable.Schema()
1280 # parameter info section
1281 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1282 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1283 size=len(lutFilterNameString))
1284 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1285 size=len(fitBandString))
1286 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1287 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1288 doc='Step units for ln(AOD) slope')
1289 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1290 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1291 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1292 doc='Step units for ln(pwv) slope')
1293 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1294 doc='Step units for ln(pwv) quadratic term')
1295 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1296 doc='Step units for global ln(pwv) parameters')
1297 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1298 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1299 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1300 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1301 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1303 # parameter section
1304 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1305 size=pars['PARALPHA'].size)
1306 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1307 size=pars['PARO3'].size)
1308 parSchema.addField('parLnTauIntercept', type='ArrayD',
1309 doc='ln(Tau) intercept parameter vector',
1310 size=pars['PARLNTAUINTERCEPT'].size)
1311 parSchema.addField('parLnTauSlope', type='ArrayD',
1312 doc='ln(Tau) slope parameter vector',
1313 size=pars['PARLNTAUSLOPE'].size)
1314 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1315 size=pars['PARLNPWVINTERCEPT'].size)
1316 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1317 size=pars['PARLNPWVSLOPE'].size)
1318 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1319 size=pars['PARLNPWVQUADRATIC'].size)
1320 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1321 size=pars['PARQESYSINTERCEPT'].size)
1322 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1323 size=pars[0]['COMPQESYSSLOPE'].size)
1324 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1325 size=pars['PARFILTEROFFSET'].size)
1326 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1327 size=pars['PARFILTEROFFSETFITFLAG'].size)
1328 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1329 doc='Global scale for retrieved ln(pwv)')
1330 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1331 doc='Global offset for retrieved ln(pwv)')
1332 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1333 doc='Nightly offset for retrieved ln(pwv)',
1334 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1335 parSchema.addField('compAbsThroughput', type='ArrayD',
1336 doc='Absolute throughput (relative to transmission curves)',
1337 size=pars['COMPABSTHROUGHPUT'].size)
1338 parSchema.addField('compRefOffset', type='ArrayD',
1339 doc='Offset between reference stars and calibrated stars',
1340 size=pars['COMPREFOFFSET'].size)
1341 parSchema.addField('compRefSigma', type='ArrayD',
1342 doc='Width of reference star/calibrated star distribution',
1343 size=pars['COMPREFSIGMA'].size)
1344 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1345 doc='Computed mirror chromaticity terms',
1346 size=pars['COMPMIRRORCHROMATICITY'].size)
1347 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1348 doc='Mirror chromaticity pivot mjd',
1349 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1350 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1351 size=pars['COMPAPERCORRPIVOT'].size)
1352 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1353 size=pars['COMPAPERCORRSLOPE'].size)
1354 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1355 size=pars['COMPAPERCORRSLOPEERR'].size)
1356 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1357 size=pars['COMPAPERCORRRANGE'].size)
1358 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1359 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1360 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1361 size=pars['COMPMODELERRFWHMPIVOT'].size)
1362 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1363 size=pars['COMPMODELERRSKYPIVOT'].size)
1364 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1365 size=pars['COMPMODELERRPARS'].size)
1366 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1367 size=pars['COMPEXPGRAY'].size)
1368 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1369 size=pars['COMPVARGRAY'].size)
1370 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1371 doc='Computed number of good stars per exposure',
1372 size=pars['COMPNGOODSTARPEREXP'].size)
1373 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1374 size=pars['COMPSIGFGCM'].size)
1375 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1376 size=pars['COMPSIGMACAL'].size)
1377 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1378 size=pars['COMPRETRIEVEDLNPWV'].size)
1379 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1380 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1381 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1382 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1383 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1384 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1385 # superstarflat section
1386 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1387 size=4)
1388 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1389 size=parSuperStarFlat.size)
1391 return parSchema
1393 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1394 lutFilterNameString, fitBandString):
1395 """
1396 Make the FGCM parameter catalog for persistence
1398 Parameters
1399 ----------
1400 parSchema: `lsst.afw.table.Schema`
1401 Parameter catalog schema
1402 pars: `numpy.ndarray`
1403 FGCM parameters to put into parCat
1404 parSuperStarFlat: `numpy.array`
1405 FGCM superstar flat array to put into parCat
1406 lutFilterNameString: `str`
1407 Combined string of all the lutFilterNames
1408 fitBandString: `str`
1409 Combined string of all the fitBands
1411 Returns
1412 -------
1413 parCat: `afwTable.BasicCatalog`
1414 Atmosphere and instrumental model parameter catalog for persistence
1415 """
1417 parCat = afwTable.BaseCatalog(parSchema)
1418 parCat.reserve(1)
1420 # The parameter catalog just has one row, with many columns for all the
1421 # atmosphere and instrument fit parameters
1422 rec = parCat.addNew()
1424 # info section
1425 rec['nCcd'] = parInfo['NCCD']
1426 rec['lutFilterNames'] = lutFilterNameString
1427 rec['fitBands'] = fitBandString
1428 # note these are not currently supported here.
1429 rec['hasExternalPwv'] = 0
1430 rec['hasExternalTau'] = 0
1432 # parameter section
1434 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1436 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1437 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1438 'parQeSysIntercept', 'compQeSysSlope',
1439 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1440 'parFilterOffset', 'parFilterOffsetFitFlag',
1441 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1442 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1443 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1444 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1445 'compModelErrSkyPivot', 'compModelErrPars',
1446 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1447 'compSigmaCal',
1448 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1449 'compRetrievedTauNight']
1451 for scalarName in scalarNames:
1452 rec[scalarName] = pars[scalarName.upper()]
1454 for arrName in arrNames:
1455 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1457 # superstar section
1458 rec['superstarSize'][:] = parSuperStarFlat.shape
1459 rec['superstar'][:] = parSuperStarFlat.flatten()
1461 return parCat
1463 def _makeFlagStarSchema(self):
1464 """
1465 Make the flagged-stars schema
1467 Returns
1468 -------
1469 flagStarSchema: `lsst.afw.table.Schema`
1470 """
1472 flagStarSchema = afwTable.Schema()
1474 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1475 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1477 return flagStarSchema
1479 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1480 """
1481 Make the flagged star catalog for persistence
1483 Parameters
1484 ----------
1485 flagStarSchema: `lsst.afw.table.Schema`
1486 Flagged star schema
1487 flagStarStruct: `numpy.ndarray`
1488 Flagged star structure from fgcm
1490 Returns
1491 -------
1492 flagStarCat: `lsst.afw.table.BaseCatalog`
1493 Flagged star catalog for persistence
1494 """
1496 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1497 flagStarCat.resize(flagStarStruct.size)
1499 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1500 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1502 return flagStarCat