Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44import lsst.afw.table as afwTable
46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47from .utilities import extractReferenceMags
48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50from .sedterms import SedboundarytermDict, SedtermDict
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57class FgcmFitCycleConfig(pexConfig.Config):
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc="Bands to run calibration",
62 dtype=str,
63 default=[],
64 )
65 fitFlag = pexConfig.ListField(
66 doc=("Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
70 dtype=int,
71 default=(0,),
72 optional=True,
73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
74 "It will be removed after v20. Use fitBands instead."),
75 )
76 fitBands = pexConfig.ListField(
77 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
78 "the atmosphere constrained from the 'fitBands' on the same night. "
79 "Must be a subset of `config.bands`"),
80 dtype=str,
81 default=[],
82 )
83 requiredFlag = pexConfig.ListField(
84 doc=("Flag for which bands are required for a star to be considered a calibration "
85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86 "be same length as config.bands, and matched band-by-band."),
87 dtype=int,
88 default=(0,),
89 optional=True,
90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
91 "It will be removed after v20. Use requiredBands instead."),
92 )
93 requiredBands = pexConfig.ListField(
94 doc=("Bands that are required for a star to be considered a calibration star. "
95 "Must be a subset of `config.bands`"),
96 dtype=str,
97 default=[],
98 )
99 filterMap = pexConfig.DictField(
100 doc="Mapping from 'filterName' to band.",
101 keytype=str,
102 itemtype=str,
103 default={},
104 )
105 doReferenceCalibration = pexConfig.Field(
106 doc="Use reference catalog as additional constraint on calibration",
107 dtype=bool,
108 default=True,
109 )
110 refStarSnMin = pexConfig.Field(
111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
112 dtype=float,
113 default=50.0,
114 )
115 refStarOutlierNSig = pexConfig.Field(
116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
118 dtype=float,
119 default=4.0,
120 )
121 applyRefStarColorCuts = pexConfig.Field(
122 doc="Apply color cuts to reference stars?",
123 dtype=bool,
124 default=True,
125 )
126 nCore = pexConfig.Field(
127 doc="Number of cores to use",
128 dtype=int,
129 default=4,
130 )
131 nStarPerRun = pexConfig.Field(
132 doc="Number of stars to run in each chunk",
133 dtype=int,
134 default=200000,
135 )
136 nExpPerRun = pexConfig.Field(
137 doc="Number of exposures to run in each chunk",
138 dtype=int,
139 default=1000,
140 )
141 reserveFraction = pexConfig.Field(
142 doc="Fraction of stars to reserve for testing",
143 dtype=float,
144 default=0.1,
145 )
146 freezeStdAtmosphere = pexConfig.Field(
147 doc="Freeze atmosphere parameters to standard (for testing)",
148 dtype=bool,
149 default=False,
150 )
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc="Precompute superstar flat for initial cycle",
153 dtype=bool,
154 default=False,
155 )
156 superStarSubCcd = pexConfig.Field(
157 doc="Compute superstar flat on sub-ccd scale",
158 dtype=bool,
159 default=True,
160 optional=True,
161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
162 "It will be removed after v20. Use superStarSubCcdDict instead."),
163 )
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166 "Must have one entry per band."),
167 keytype=str,
168 itemtype=bool,
169 default={},
170 )
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173 "Global default is first-order polynomials, and should be overridden "
174 "on a camera-by-camera basis depending on the ISR."),
175 dtype=int,
176 default=1,
177 )
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
180 "suppress high-order cross terms?"),
181 dtype=bool,
182 default=False,
183 )
184 superStarSigmaClip = pexConfig.Field(
185 doc="Number of sigma to clip outliers when selecting for superstar flats",
186 dtype=float,
187 default=5.0,
188 )
189 ccdGraySubCcd = pexConfig.Field(
190 doc="Compute CCD gray terms on sub-ccd scale",
191 dtype=bool,
192 default=False,
193 optional=True,
194 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
195 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
196 )
197 ccdGraySubCcdDict = pexConfig.DictField(
198 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
199 "('ccd gray') on a sub-ccd scale."),
200 keytype=str,
201 itemtype=bool,
202 default={},
203 )
204 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
206 dtype=int,
207 default=1,
208 )
209 ccdGraySubCcdTriangular = pexConfig.Field(
210 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
211 "suppress high-order cross terms?"),
212 dtype=bool,
213 default=True,
214 )
215 cycleNumber = pexConfig.Field(
216 doc=("FGCM fit cycle number. This is automatically incremented after each run "
217 "and stage of outlier rejection. See cookbook for details."),
218 dtype=int,
219 default=None,
220 )
221 isFinalCycle = pexConfig.Field(
222 doc=("Is this the final cycle of the fitting? Will automatically compute final "
223 "selection of stars and photometric exposures, and will output zeropoints "
224 "and standard stars for use in fgcmOutputProducts"),
225 dtype=bool,
226 default=False,
227 )
228 maxIterBeforeFinalCycle = pexConfig.Field(
229 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
230 "will always be 0 in the final cycle for cleanup and final selection."),
231 dtype=int,
232 default=50,
233 )
234 utBoundary = pexConfig.Field(
235 doc="Boundary (in UTC) from day-to-day",
236 dtype=float,
237 default=None,
238 )
239 washMjds = pexConfig.ListField(
240 doc="Mirror wash MJDs",
241 dtype=float,
242 default=(0.0,),
243 )
244 epochMjds = pexConfig.ListField(
245 doc="Epoch boundaries in MJD",
246 dtype=float,
247 default=(0.0,),
248 )
249 minObsPerBand = pexConfig.Field(
250 doc="Minimum good observations per band",
251 dtype=int,
252 default=2,
253 )
254 # TODO: When DM-16511 is done, it will be possible to get the
255 # telescope latitude directly from the camera.
256 latitude = pexConfig.Field(
257 doc="Observatory latitude",
258 dtype=float,
259 default=None,
260 )
261 brightObsGrayMax = pexConfig.Field(
262 doc="Maximum gray extinction to be considered bright observation",
263 dtype=float,
264 default=0.15,
265 )
266 minStarPerCcd = pexConfig.Field(
267 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
268 "CCDs with fewer stars will have their calibration estimated from other "
269 "CCDs in the same visit, with zeropoint error increased accordingly."),
270 dtype=int,
271 default=5,
272 )
273 minCcdPerExp = pexConfig.Field(
274 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
275 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
276 dtype=int,
277 default=5,
278 )
279 maxCcdGrayErr = pexConfig.Field(
280 doc="Maximum error on CCD gray offset to be considered photometric",
281 dtype=float,
282 default=0.05,
283 )
284 minStarPerExp = pexConfig.Field(
285 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
286 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
287 dtype=int,
288 default=600,
289 )
290 minExpPerNight = pexConfig.Field(
291 doc="Minimum number of good exposures/visits to consider a partly photometric night",
292 dtype=int,
293 default=10,
294 )
295 expGrayInitialCut = pexConfig.Field(
296 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
297 "observations."),
298 dtype=float,
299 default=-0.25,
300 )
301 expGrayPhotometricCut = pexConfig.ListField(
302 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
303 "Must be same length as config.bands, and matched band-by-band."),
304 dtype=float,
305 default=(0.0,),
306 optional=True,
307 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
308 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
309 )
310 expGrayPhotometricCutDict = pexConfig.DictField(
311 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
312 "('gray term') for a visit to be considered photometric. Must have one "
313 "entry per band. Broad-band filters should be -0.05."),
314 keytype=str,
315 itemtype=float,
316 default={},
317 )
318 expGrayHighCut = pexConfig.ListField(
319 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
320 "Must be same length as config.bands, and matched band-by-band."),
321 dtype=float,
322 default=(0.0,),
323 optional=True,
324 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
325 "It will be removed after v20. Use expGrayHighCutDict instead."),
326 )
327 expGrayHighCutDict = pexConfig.DictField(
328 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
329 "('gray term') for a visit to be considered photometric. Must have one "
330 "entry per band. Broad-band filters should be 0.2."),
331 keytype=str,
332 itemtype=float,
333 default={},
334 )
335 expGrayRecoverCut = pexConfig.Field(
336 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
337 "Visits with more gray extinction will only get CCD zeropoints if there are "
338 "sufficient star observations (minStarPerCcd) on that CCD."),
339 dtype=float,
340 default=-1.0,
341 )
342 expVarGrayPhotometricCut = pexConfig.Field(
343 doc="Maximum exposure variance to be considered possibly photometric",
344 dtype=float,
345 default=0.0005,
346 optional=True,
347 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
348 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
349 )
350 expVarGrayPhotometricCutDict = pexConfig.DictField(
351 doc=("Per-band specification on maximum exposure variance to be considered possibly "
352 "photometric. Must have one entry per band. Broad-band filters should be "
353 "0.0005."),
354 keytype=str,
355 itemtype=float,
356 default={},
357 )
358 expGrayErrRecoverCut = pexConfig.Field(
359 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
360 "Visits with more gray variance will only get CCD zeropoints if there are "
361 "sufficient star observations (minStarPerCcd) on that CCD."),
362 dtype=float,
363 default=0.05,
364 )
365 aperCorrFitNBins = pexConfig.Field(
366 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
367 "no fit will be performed, and the config.aperCorrInputSlopes will be "
368 "used if available."),
369 dtype=int,
370 default=10,
371 )
372 aperCorrInputSlopes = pexConfig.ListField(
373 doc=("Aperture correction input slope parameters. These are used on the first "
374 "fit iteration, and aperture correction parameters will be updated from "
375 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
376 "when there is insufficient data to fit the parameters (e.g. tract mode). "
377 "If set, must be same length as config.bands, and matched band-by-band."),
378 dtype=float,
379 default=[],
380 optional=True,
381 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
382 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
383 )
384 aperCorrInputSlopeDict = pexConfig.DictField(
385 doc=("Per-band specification of aperture correction input slope parameters. These "
386 "are used on the first fit iteration, and aperture correction parameters will "
387 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
388 "to set this when there is insufficient data to fit the parameters (e.g. "
389 "tract mode)."),
390 keytype=str,
391 itemtype=float,
392 default={},
393 )
394 sedFudgeFactors = pexConfig.ListField(
395 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
396 "config.bands, and matched band-by-band."),
397 dtype=float,
398 default=(0,),
399 optional=True,
400 deprecated=("This field has been deprecated and will be removed after v20. "
401 "Please use sedSlopeTermMap and sedSlopeMap."),
402 )
403 sedboundaryterms = pexConfig.ConfigField(
404 doc="Mapping from bands to SED boundary term names used is sedterms.",
405 dtype=SedboundarytermDict,
406 )
407 sedterms = pexConfig.ConfigField(
408 doc="Mapping from terms to bands for fgcm linear SED approximations.",
409 dtype=SedtermDict,
410 )
411 sigFgcmMaxErr = pexConfig.Field(
412 doc="Maximum mag error for fitting sigma_FGCM",
413 dtype=float,
414 default=0.01,
415 )
416 sigFgcmMaxEGray = pexConfig.ListField(
417 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
418 "May be 1 element (same for all bands) or the same length as config.bands."),
419 dtype=float,
420 default=(0.05,),
421 optional=True,
422 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
423 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
424 )
425 sigFgcmMaxEGrayDict = pexConfig.DictField(
426 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
427 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
428 "should be 0.05."),
429 keytype=str,
430 itemtype=float,
431 default={},
432 )
433 ccdGrayMaxStarErr = pexConfig.Field(
434 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
435 "computation"),
436 dtype=float,
437 default=0.10,
438 )
439 approxThroughput = pexConfig.ListField(
440 doc=("Approximate overall throughput at start of calibration observations. "
441 "May be 1 element (same for all bands) or the same length as config.bands."),
442 dtype=float,
443 default=(1.0, ),
444 optional=True,
445 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
446 "It will be removed after v20. Use approxThroughputDict instead."),
447 )
448 approxThroughputDict = pexConfig.DictField(
449 doc=("Per-band specification of the approximate overall throughput at the start of "
450 "calibration observations. Must have one entry per band. Typically should "
451 "be 1.0."),
452 keytype=str,
453 itemtype=float,
454 default={},
455 )
456 sigmaCalRange = pexConfig.ListField(
457 doc="Allowed range for systematic error floor estimation",
458 dtype=float,
459 default=(0.001, 0.003),
460 )
461 sigmaCalFitPercentile = pexConfig.ListField(
462 doc="Magnitude percentile range to fit systematic error floor",
463 dtype=float,
464 default=(0.05, 0.15),
465 )
466 sigmaCalPlotPercentile = pexConfig.ListField(
467 doc="Magnitude percentile range to plot systematic error floor",
468 dtype=float,
469 default=(0.05, 0.95),
470 )
471 sigma0Phot = pexConfig.Field(
472 doc="Systematic error floor for all zeropoints",
473 dtype=float,
474 default=0.003,
475 )
476 mapLongitudeRef = pexConfig.Field(
477 doc="Reference longitude for plotting maps",
478 dtype=float,
479 default=0.0,
480 )
481 mapNSide = pexConfig.Field(
482 doc="Healpix nside for plotting maps",
483 dtype=int,
484 default=256,
485 )
486 outfileBase = pexConfig.Field(
487 doc="Filename start for plot output files",
488 dtype=str,
489 default=None,
490 )
491 starColorCuts = pexConfig.ListField(
492 doc="Encoded star-color cuts (to be cleaned up)",
493 dtype=str,
494 default=("NO_DATA",),
495 )
496 colorSplitIndices = pexConfig.ListField(
497 doc="Band indices to use to split stars by color",
498 dtype=int,
499 default=None,
500 optional=True,
501 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
502 "It will be removed after v20. Use colorSplitBands instead."),
503 )
504 colorSplitBands = pexConfig.ListField(
505 doc="Band names to use to split stars by color. Must have 2 entries.",
506 dtype=str,
507 length=2,
508 default=('g', 'i'),
509 )
510 modelMagErrors = pexConfig.Field(
511 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
512 dtype=bool,
513 default=True,
514 )
515 useQuadraticPwv = pexConfig.Field(
516 doc="Model PWV with a quadratic term for variation through the night?",
517 dtype=bool,
518 default=False,
519 )
520 instrumentParsPerBand = pexConfig.Field(
521 doc=("Model instrumental parameters per band? "
522 "Otherwise, instrumental parameters (QE changes with time) are "
523 "shared among all bands."),
524 dtype=bool,
525 default=False,
526 )
527 instrumentSlopeMinDeltaT = pexConfig.Field(
528 doc=("Minimum time change (in days) between observations to use in constraining "
529 "instrument slope."),
530 dtype=float,
531 default=20.0,
532 )
533 fitMirrorChromaticity = pexConfig.Field(
534 doc="Fit (intraband) mirror chromatic term?",
535 dtype=bool,
536 default=False,
537 )
538 coatingMjds = pexConfig.ListField(
539 doc="Mirror coating dates in MJD",
540 dtype=float,
541 default=(0.0,),
542 )
543 outputStandardsBeforeFinalCycle = pexConfig.Field(
544 doc="Output standard stars prior to final cycle? Used in debugging.",
545 dtype=bool,
546 default=False,
547 )
548 outputZeropointsBeforeFinalCycle = pexConfig.Field(
549 doc="Output standard stars prior to final cycle? Used in debugging.",
550 dtype=bool,
551 default=False,
552 )
553 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
554 doc=("Use star repeatability (instead of exposures) for computing photometric "
555 "cuts? Recommended for tract mode or bands with few exposures. "
556 "May be 1 element (same for all bands) or the same length as config.bands."),
557 dtype=bool,
558 default=(False,),
559 optional=True,
560 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
561 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
562 )
563 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
564 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
565 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
566 keytype=str,
567 itemtype=bool,
568 default={},
569 )
570 autoPhotometricCutNSig = pexConfig.Field(
571 doc=("Number of sigma for automatic computation of (low) photometric cut. "
572 "Cut is based on exposure gray width (per band), unless "
573 "useRepeatabilityForExpGrayCuts is set, in which case the star "
574 "repeatability is used (also per band)."),
575 dtype=float,
576 default=3.0,
577 )
578 autoHighCutNSig = pexConfig.Field(
579 doc=("Number of sigma for automatic computation of (high) outlier cut. "
580 "Cut is based on exposure gray width (per band), unless "
581 "useRepeatabilityForExpGrayCuts is set, in which case the star "
582 "repeatability is used (also per band)."),
583 dtype=float,
584 default=4.0,
585 )
586 quietMode = pexConfig.Field(
587 doc="Be less verbose with logging.",
588 dtype=bool,
589 default=False,
590 )
592 def setDefaults(self):
593 pass
595 def validate(self):
596 super().validate()
598 for band in self.fitBands:
599 if band not in self.bands:
600 msg = 'fitBand %s not in bands' % (band)
601 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
602 for band in self.requiredBands:
603 if band not in self.bands:
604 msg = 'requiredBand %s not in bands' % (band)
605 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
606 for band in self.colorSplitBands:
607 if band not in self.bands:
608 msg = 'colorSplitBand %s not in bands' % (band)
609 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
610 for band in self.bands:
611 if band not in self.superStarSubCcdDict:
612 msg = 'band %s not in superStarSubCcdDict' % (band)
613 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
614 self, msg)
615 if band not in self.ccdGraySubCcdDict:
616 msg = 'band %s not in ccdGraySubCcdDict' % (band)
617 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
618 self, msg)
619 if band not in self.expGrayPhotometricCutDict:
620 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
621 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
622 self, msg)
623 if band not in self.expGrayHighCutDict:
624 msg = 'band %s not in expGrayHighCutDict' % (band)
625 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
626 self, msg)
627 if band not in self.expVarGrayPhotometricCutDict:
628 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
629 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
630 self, msg)
631 if band not in self.sigFgcmMaxEGrayDict:
632 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
633 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
634 self, msg)
635 if band not in self.approxThroughputDict:
636 msg = 'band %s not in approxThroughputDict' % (band)
637 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
638 self, msg)
639 if band not in self.useRepeatabilityForExpGrayCutsDict:
640 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
641 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
642 self, msg)
645class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
646 """Subclass of TaskRunner for fgcmFitCycleTask
648 fgcmFitCycleTask.run() takes one argument, the butler, and uses
649 stars and visits previously extracted from dataRefs by
650 fgcmBuildStars.
651 This Runner does not perform any dataRef parallelization, but the FGCM
652 code called by the Task uses python multiprocessing (see the "ncores"
653 config option).
654 """
656 @staticmethod
657 def getTargetList(parsedCmd):
658 """
659 Return a list with one element, the butler.
660 """
661 return [parsedCmd.butler]
663 def __call__(self, butler):
664 """
665 Parameters
666 ----------
667 butler: `lsst.daf.persistence.Butler`
669 Returns
670 -------
671 exitStatus: `list` with `pipeBase.Struct`
672 exitStatus (0: success; 1: failure)
673 """
675 task = self.TaskClass(config=self.config, log=self.log)
677 exitStatus = 0
678 if self.doRaise:
679 task.runDataRef(butler)
680 else:
681 try:
682 task.runDataRef(butler)
683 except Exception as e:
684 exitStatus = 1
685 task.log.fatal("Failed: %s" % e)
686 if not isinstance(e, pipeBase.TaskError):
687 traceback.print_exc(file=sys.stderr)
689 task.writeMetadata(butler)
691 # The task does not return any results:
692 return [pipeBase.Struct(exitStatus=exitStatus)]
694 def run(self, parsedCmd):
695 """
696 Run the task, with no multiprocessing
698 Parameters
699 ----------
700 parsedCmd: ArgumentParser parsed command line
701 """
703 resultList = []
705 if self.precall(parsedCmd):
706 targetList = self.getTargetList(parsedCmd)
707 # make sure that we only get 1
708 resultList = self(targetList[0])
710 return resultList
713class FgcmFitCycleTask(pipeBase.CmdLineTask):
714 """
715 Run Single fit cycle for FGCM global calibration
716 """
718 ConfigClass = FgcmFitCycleConfig
719 RunnerClass = FgcmFitCycleRunner
720 _DefaultName = "fgcmFitCycle"
722 def __init__(self, butler=None, **kwargs):
723 """
724 Instantiate an fgcmFitCycle.
726 Parameters
727 ----------
728 butler : `lsst.daf.persistence.Butler`
729 """
731 pipeBase.CmdLineTask.__init__(self, **kwargs)
733 # no saving of metadata for now
734 def _getMetadataName(self):
735 return None
737 @pipeBase.timeMethod
738 def runDataRef(self, butler):
739 """
740 Run a single fit cycle for FGCM
742 Parameters
743 ----------
744 butler: `lsst.daf.persistence.Butler`
745 """
747 self._fgcmFitCycle(butler)
749 def writeConfig(self, butler, clobber=False, doBackup=True):
750 """Write the configuration used for processing the data, or check that an existing
751 one is equal to the new one if present. This is an override of the regular
752 version from pipe_base that knows about fgcmcycle.
754 Parameters
755 ----------
756 butler : `lsst.daf.persistence.Butler`
757 Data butler used to write the config. The config is written to dataset type
758 `CmdLineTask._getConfigName`.
759 clobber : `bool`, optional
760 A boolean flag that controls what happens if a config already has been saved:
761 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
762 - `False`: raise `TaskError` if this config does not match the existing config.
763 doBackup : `bool`, optional
764 Set to `True` to backup the config files if clobbering.
765 """
766 configName = self._getConfigName()
767 if configName is None:
768 return
769 if clobber:
770 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
771 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
772 # this may be subject to a race condition; see #2789
773 try:
774 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
775 except Exception as exc:
776 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
777 (configName, exc))
779 def logConfigMismatch(msg):
780 self.log.fatal("Comparing configuration: %s", msg)
782 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
783 raise pipeBase.TaskError(
784 ("Config does not match existing task config %r on disk; tasks configurations " +
785 "must be consistent within the same output repo (override with --clobber-config)") %
786 (configName,))
787 else:
788 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
790 def _fgcmFitCycle(self, butler):
791 """
792 Run the fit cycle
794 Parameters
795 ----------
796 butler: `lsst.daf.persistence.Butler`
797 """
799 self._checkDatasetsExist(butler)
801 # Set defaults on whether to output standards and zeropoints
802 self.maxIter = self.config.maxIterBeforeFinalCycle
803 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
804 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
805 self.resetFitParameters = True
807 if self.config.isFinalCycle:
808 # This is the final fit cycle, so we do not want to reset fit
809 # parameters, we want to run a final "clean-up" with 0 fit iterations,
810 # and we always want to output standards and zeropoints
811 self.maxIter = 0
812 self.outputStandards = True
813 self.outputZeropoints = True
814 self.resetFitParameters = False
816 camera = butler.get('camera')
817 configDict = makeConfigDict(self.config, self.log, camera,
818 self.maxIter, self.resetFitParameters,
819 self.outputZeropoints)
821 lutCat = butler.get('fgcmLookUpTable')
822 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
823 del lutCat
825 # next we need the exposure/visit information
827 # fgcmExpInfo = self._loadVisitCatalog(butler)
828 visitCat = butler.get('fgcmVisitCatalog')
829 fgcmExpInfo = translateVisitCatalog(visitCat)
830 del visitCat
832 # Use the first orientation.
833 # TODO: DM-21215 will generalize to arbitrary camera orientations
834 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
836 noFitsDict = {'lutIndex': lutIndexVals,
837 'lutStd': lutStd,
838 'expInfo': fgcmExpInfo,
839 'ccdOffsets': ccdOffsets}
841 # set up the fitter object
842 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
843 noFitsDict=noFitsDict, noOutput=True)
845 # create the parameter object
846 if (fgcmFitCycle.initialCycle):
847 # cycle = 0, initial cycle
848 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
849 fgcmLut,
850 fgcmExpInfo)
851 else:
852 inParInfo, inParams, inSuperStar = self._loadParameters(butler)
853 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
854 fgcmExpInfo,
855 inParInfo,
856 inParams,
857 inSuperStar)
859 lastCycle = configDict['cycleNumber'] - 1
861 # set up the stars...
862 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
864 starObs = butler.get('fgcmStarObservations')
865 starIds = butler.get('fgcmStarIds')
866 starIndices = butler.get('fgcmStarIndices')
868 # grab the flagged stars if available
869 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
870 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
871 flagId = flaggedStars['objId'][:]
872 flagFlag = flaggedStars['objFlag'][:]
873 else:
874 flagId = None
875 flagFlag = None
877 if self.config.doReferenceCalibration:
878 refStars = butler.get('fgcmReferenceStars')
880 refMag, refMagErr = extractReferenceMags(refStars,
881 self.config.bands,
882 self.config.filterMap)
883 refId = refStars['fgcm_id'][:]
884 else:
885 refId = None
886 refMag = None
887 refMagErr = None
889 # match star observations to visits
890 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
891 # actually be transferred into fgcm using the indexing below.
892 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
894 # The fgcmStars.loadStars method will copy all the star information into
895 # special shared memory objects that will not blow up the memory usage when
896 # used with python multiprocessing. Once all the numbers are copied,
897 # it is necessary to release all references to the objects that previously
898 # stored the data to ensure that the garbage collector can clear the memory,
899 # and ensure that this memory is not copied when multiprocessing kicks in.
901 # We determine the conversion from the native units (typically radians) to
902 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
903 # numpy arrays rather than Angles, which would we approximately 600x slower.
904 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
906 fgcmStars.loadStars(fgcmPars,
907 starObs['visit'][starIndices['obsIndex']],
908 starObs['ccd'][starIndices['obsIndex']],
909 starObs['ra'][starIndices['obsIndex']] * conv,
910 starObs['dec'][starIndices['obsIndex']] * conv,
911 starObs['instMag'][starIndices['obsIndex']],
912 starObs['instMagErr'][starIndices['obsIndex']],
913 fgcmExpInfo['FILTERNAME'][visitIndex],
914 starIds['fgcm_id'][:],
915 starIds['ra'][:],
916 starIds['dec'][:],
917 starIds['obsArrIndex'][:],
918 starIds['nObs'][:],
919 obsX=starObs['x'][starIndices['obsIndex']],
920 obsY=starObs['y'][starIndices['obsIndex']],
921 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
922 refID=refId,
923 refMag=refMag,
924 refMagErr=refMagErr,
925 flagID=flagId,
926 flagFlag=flagFlag,
927 computeNobs=True)
929 # Release all references to temporary objects holding star data (see above)
930 starObs = None
931 starIds = None
932 starIndices = None
933 flagId = None
934 flagFlag = None
935 flaggedStars = None
936 refStars = None
938 # and set the bits in the cycle object
939 fgcmFitCycle.setLUT(fgcmLut)
940 fgcmFitCycle.setStars(fgcmStars)
941 fgcmFitCycle.setPars(fgcmPars)
943 # finish the setup
944 fgcmFitCycle.finishSetup()
946 # and run
947 fgcmFitCycle.run()
949 ##################
950 # Persistance
951 ##################
953 self._persistFgcmDatasets(butler, fgcmFitCycle)
955 # Output the config for the next cycle
956 # We need to make a copy since the input one has been frozen
958 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
959 i, b in enumerate(self.config.bands)}
960 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
961 i, band in enumerate(self.config.bands)}
963 outConfig = copy.copy(self.config)
964 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
965 precomputeSuperStarInitialCycle=False,
966 freezeStdAtmosphere=False,
967 expGrayPhotometricCutDict=updatedPhotometricCutDict,
968 expGrayHighCutDict=updatedHighCutDict)
969 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
970 outConfig.cycleNumber)
971 outConfig.save(configFileName)
973 if self.config.isFinalCycle == 1:
974 # We are done, ready to output products
975 self.log.info("Everything is in place to run fgcmOutputProducts.py")
976 else:
977 self.log.info("Saved config for next cycle to %s" % (configFileName))
978 self.log.info("Be sure to look at:")
979 self.log.info(" config.expGrayPhotometricCut")
980 self.log.info(" config.expGrayHighCut")
981 self.log.info("If you are satisfied with the fit, please set:")
982 self.log.info(" config.isFinalCycle = True")
984 def _checkDatasetsExist(self, butler):
985 """
986 Check if necessary datasets exist to run fgcmFitCycle
988 Parameters
989 ----------
990 butler: `lsst.daf.persistence.Butler`
992 Raises
993 ------
994 RuntimeError
995 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
996 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
997 If cycleNumber > 0, then also checks for fgcmFitParameters,
998 fgcmFlaggedStars.
999 """
1001 if not butler.datasetExists('fgcmVisitCatalog'):
1002 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1003 if not butler.datasetExists('fgcmStarObservations'):
1004 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1005 if not butler.datasetExists('fgcmStarIds'):
1006 raise RuntimeError("Could not find fgcmStarIds in repo!")
1007 if not butler.datasetExists('fgcmStarIndices'):
1008 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1009 if not butler.datasetExists('fgcmLookUpTable'):
1010 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1012 # Need additional datasets if we are not the initial cycle
1013 if (self.config.cycleNumber > 0):
1014 if not butler.datasetExists('fgcmFitParameters',
1015 fgcmcycle=self.config.cycleNumber-1):
1016 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1017 (self.config.cycleNumber-1))
1018 if not butler.datasetExists('fgcmFlaggedStars',
1019 fgcmcycle=self.config.cycleNumber-1):
1020 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1021 (self.config.cycleNumber-1))
1023 # And additional dataset if we want reference calibration
1024 if self.config.doReferenceCalibration:
1025 if not butler.datasetExists('fgcmReferenceStars'):
1026 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1027 "doReferenceCalibration is True.")
1029 def _loadParameters(self, butler):
1030 """
1031 Load FGCM parameters from a previous fit cycle
1033 Parameters
1034 ----------
1035 butler: `lsst.daf.persistence.Butler`
1037 Returns
1038 -------
1039 inParInfo: `numpy.ndarray`
1040 Numpy array parameter information formatted for input to fgcm
1041 inParameters: `numpy.ndarray`
1042 Numpy array parameter values formatted for input to fgcm
1043 inSuperStar: `numpy.array`
1044 Superstar flat formatted for input to fgcm
1045 """
1047 # note that we already checked that this is available
1048 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1050 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1051 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1052 parNotFitBands = np.array(parCat[0]['notFitBands'].split(','))
1054 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1055 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1056 (parLutFilterNames.size, )),
1057 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1058 ('NOTFITBANDS', parNotFitBands.dtype.str, (parNotFitBands.size, )),
1059 ('LNTAUUNIT', 'f8'),
1060 ('LNTAUSLOPEUNIT', 'f8'),
1061 ('ALPHAUNIT', 'f8'),
1062 ('LNPWVUNIT', 'f8'),
1063 ('LNPWVSLOPEUNIT', 'f8'),
1064 ('LNPWVQUADRATICUNIT', 'f8'),
1065 ('LNPWVGLOBALUNIT', 'f8'),
1066 ('O3UNIT', 'f8'),
1067 ('QESYSUNIT', 'f8'),
1068 ('FILTEROFFSETUNIT', 'f8'),
1069 ('HASEXTERNALPWV', 'i2'),
1070 ('HASEXTERNALTAU', 'i2')])
1071 inParInfo['NCCD'] = parCat['nCcd']
1072 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1073 inParInfo['FITBANDS'][:] = parFitBands
1074 inParInfo['NOTFITBANDS'][:] = parNotFitBands
1075 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1076 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1078 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1079 ('PARO3', 'f8', (parCat['parO3'].size, )),
1080 ('PARLNTAUINTERCEPT', 'f8',
1081 (parCat['parLnTauIntercept'].size, )),
1082 ('PARLNTAUSLOPE', 'f8',
1083 (parCat['parLnTauSlope'].size, )),
1084 ('PARLNPWVINTERCEPT', 'f8',
1085 (parCat['parLnPwvIntercept'].size, )),
1086 ('PARLNPWVSLOPE', 'f8',
1087 (parCat['parLnPwvSlope'].size, )),
1088 ('PARLNPWVQUADRATIC', 'f8',
1089 (parCat['parLnPwvQuadratic'].size, )),
1090 ('PARQESYSINTERCEPT', 'f8',
1091 (parCat['parQeSysIntercept'].size, )),
1092 ('COMPQESYSSLOPE', 'f8',
1093 (parCat['compQeSysSlope'].size, )),
1094 ('PARFILTEROFFSET', 'f8',
1095 (parCat['parFilterOffset'].size, )),
1096 ('PARFILTEROFFSETFITFLAG', 'i2',
1097 (parCat['parFilterOffsetFitFlag'].size, )),
1098 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1099 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1100 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1101 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1102 ('COMPABSTHROUGHPUT', 'f8',
1103 (parCat['compAbsThroughput'].size, )),
1104 ('COMPREFOFFSET', 'f8',
1105 (parCat['compRefOffset'].size, )),
1106 ('COMPREFSIGMA', 'f8',
1107 (parCat['compRefSigma'].size, )),
1108 ('COMPMIRRORCHROMATICITY', 'f8',
1109 (parCat['compMirrorChromaticity'].size, )),
1110 ('MIRRORCHROMATICITYPIVOT', 'f8',
1111 (parCat['mirrorChromaticityPivot'].size, )),
1112 ('COMPAPERCORRPIVOT', 'f8',
1113 (parCat['compAperCorrPivot'].size, )),
1114 ('COMPAPERCORRSLOPE', 'f8',
1115 (parCat['compAperCorrSlope'].size, )),
1116 ('COMPAPERCORRSLOPEERR', 'f8',
1117 (parCat['compAperCorrSlopeErr'].size, )),
1118 ('COMPAPERCORRRANGE', 'f8',
1119 (parCat['compAperCorrRange'].size, )),
1120 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1121 (parCat['compModelErrExptimePivot'].size, )),
1122 ('COMPMODELERRFWHMPIVOT', 'f8',
1123 (parCat['compModelErrFwhmPivot'].size, )),
1124 ('COMPMODELERRSKYPIVOT', 'f8',
1125 (parCat['compModelErrSkyPivot'].size, )),
1126 ('COMPMODELERRPARS', 'f8',
1127 (parCat['compModelErrPars'].size, )),
1128 ('COMPEXPGRAY', 'f8',
1129 (parCat['compExpGray'].size, )),
1130 ('COMPVARGRAY', 'f8',
1131 (parCat['compVarGray'].size, )),
1132 ('COMPNGOODSTARPEREXP', 'i4',
1133 (parCat['compNGoodStarPerExp'].size, )),
1134 ('COMPSIGFGCM', 'f8',
1135 (parCat['compSigFgcm'].size, )),
1136 ('COMPSIGMACAL', 'f8',
1137 (parCat['compSigmaCal'].size, )),
1138 ('COMPRETRIEVEDLNPWV', 'f8',
1139 (parCat['compRetrievedLnPwv'].size, )),
1140 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1141 (parCat['compRetrievedLnPwvRaw'].size, )),
1142 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1143 (parCat['compRetrievedLnPwvFlag'].size, )),
1144 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1145 (parCat['compRetrievedTauNight'].size, ))])
1147 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1148 inParams['PARO3'][:] = parCat['parO3'][0, :]
1149 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1150 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1151 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1152 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1153 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1154 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1155 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1156 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1157 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1158 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1159 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1160 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1161 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1162 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1163 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1164 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1165 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1166 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1167 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1168 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1169 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1170 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1171 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1172 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1173 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1174 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1175 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1176 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1177 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1178 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1179 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1180 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1181 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1182 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1184 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1185 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1187 return (inParInfo, inParams, inSuperStar)
1189 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1190 """
1191 Persist FGCM datasets through the butler.
1193 Parameters
1194 ----------
1195 butler: `lsst.daf.persistence.Butler`
1196 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1197 Fgcm Fit cycle object
1198 """
1200 # Save the parameters
1201 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1203 parSchema = afwTable.Schema()
1205 comma = ','
1206 lutFilterNameString = comma.join([n.decode('utf-8')
1207 for n in parInfo['LUTFILTERNAMES'][0]])
1208 fitBandString = comma.join([n.decode('utf-8')
1209 for n in parInfo['FITBANDS'][0]])
1210 notFitBandString = comma.join([n.decode('utf-8')
1211 for n in parInfo['NOTFITBANDS'][0]])
1213 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1214 lutFilterNameString, fitBandString, notFitBandString)
1215 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1216 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1217 lutFilterNameString, fitBandString, notFitBandString)
1219 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1221 # Save the indices of the flagged stars
1222 # (stars that have been (a) reserved from the fit for testing and
1223 # (b) bad stars that have failed quality checks.)
1224 flagStarSchema = self._makeFlagStarSchema()
1225 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1226 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1228 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1230 # Save the zeropoint information and atmospheres only if desired
1231 if self.outputZeropoints:
1232 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1233 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1235 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1236 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1238 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1240 # Save atmosphere values
1241 # These are generated by the same code that generates zeropoints
1242 atmSchema = makeAtmSchema()
1243 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1245 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1247 # Save the standard stars (if configured)
1248 if self.outputStandards:
1249 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1250 stdSchema = makeStdSchema(len(goodBands))
1251 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1253 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1255 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1256 lutFilterNameString, fitBandString, notFitBandString):
1257 """
1258 Make the parameter persistence schema
1260 Parameters
1261 ----------
1262 parInfo: `numpy.ndarray`
1263 Parameter information returned by fgcm
1264 pars: `numpy.ndarray`
1265 Parameter values returned by fgcm
1266 parSuperStarFlat: `numpy.array`
1267 Superstar flat values returned by fgcm
1268 lutFilterNameString: `str`
1269 Combined string of all the lutFilterNames
1270 fitBandString: `str`
1271 Combined string of all the fitBands
1272 notFitBandString: `str`
1273 Combined string of all the bands not used in the fit
1275 Returns
1276 -------
1277 parSchema: `afwTable.schema`
1278 """
1280 parSchema = afwTable.Schema()
1282 # parameter info section
1283 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1284 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1285 size=len(lutFilterNameString))
1286 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1287 size=len(fitBandString))
1288 parSchema.addField('notFitBands', type=str, doc='Bands that were not fit',
1289 size=len(notFitBandString))
1290 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1291 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1292 doc='Step units for ln(AOD) slope')
1293 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1294 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1295 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1296 doc='Step units for ln(pwv) slope')
1297 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1298 doc='Step units for ln(pwv) quadratic term')
1299 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1300 doc='Step units for global ln(pwv) parameters')
1301 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1302 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1303 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1304 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1305 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1307 # parameter section
1308 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1309 size=pars['PARALPHA'].size)
1310 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1311 size=pars['PARO3'].size)
1312 parSchema.addField('parLnTauIntercept', type='ArrayD',
1313 doc='ln(Tau) intercept parameter vector',
1314 size=pars['PARLNTAUINTERCEPT'].size)
1315 parSchema.addField('parLnTauSlope', type='ArrayD',
1316 doc='ln(Tau) slope parameter vector',
1317 size=pars['PARLNTAUSLOPE'].size)
1318 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1319 size=pars['PARLNPWVINTERCEPT'].size)
1320 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1321 size=pars['PARLNPWVSLOPE'].size)
1322 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1323 size=pars['PARLNPWVQUADRATIC'].size)
1324 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1325 size=pars['PARQESYSINTERCEPT'].size)
1326 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1327 size=pars[0]['COMPQESYSSLOPE'].size)
1328 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1329 size=pars['PARFILTEROFFSET'].size)
1330 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1331 size=pars['PARFILTEROFFSETFITFLAG'].size)
1332 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1333 doc='Global scale for retrieved ln(pwv)')
1334 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1335 doc='Global offset for retrieved ln(pwv)')
1336 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1337 doc='Nightly offset for retrieved ln(pwv)',
1338 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1339 parSchema.addField('compAbsThroughput', type='ArrayD',
1340 doc='Absolute throughput (relative to transmission curves)',
1341 size=pars['COMPABSTHROUGHPUT'].size)
1342 parSchema.addField('compRefOffset', type='ArrayD',
1343 doc='Offset between reference stars and calibrated stars',
1344 size=pars['COMPREFOFFSET'].size)
1345 parSchema.addField('compRefSigma', type='ArrayD',
1346 doc='Width of reference star/calibrated star distribution',
1347 size=pars['COMPREFSIGMA'].size)
1348 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1349 doc='Computed mirror chromaticity terms',
1350 size=pars['COMPMIRRORCHROMATICITY'].size)
1351 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1352 doc='Mirror chromaticity pivot mjd',
1353 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1354 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1355 size=pars['COMPAPERCORRPIVOT'].size)
1356 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1357 size=pars['COMPAPERCORRSLOPE'].size)
1358 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1359 size=pars['COMPAPERCORRSLOPEERR'].size)
1360 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1361 size=pars['COMPAPERCORRRANGE'].size)
1362 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1363 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1364 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1365 size=pars['COMPMODELERRFWHMPIVOT'].size)
1366 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1367 size=pars['COMPMODELERRSKYPIVOT'].size)
1368 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1369 size=pars['COMPMODELERRPARS'].size)
1370 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1371 size=pars['COMPEXPGRAY'].size)
1372 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1373 size=pars['COMPVARGRAY'].size)
1374 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1375 doc='Computed number of good stars per exposure',
1376 size=pars['COMPNGOODSTARPEREXP'].size)
1377 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1378 size=pars['COMPSIGFGCM'].size)
1379 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1380 size=pars['COMPSIGMACAL'].size)
1381 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1382 size=pars['COMPRETRIEVEDLNPWV'].size)
1383 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1384 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1385 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1386 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1387 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1388 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1389 # superstarflat section
1390 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1391 size=4)
1392 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1393 size=parSuperStarFlat.size)
1395 return parSchema
1397 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1398 lutFilterNameString, fitBandString, notFitBandString):
1399 """
1400 Make the FGCM parameter catalog for persistence
1402 Parameters
1403 ----------
1404 parSchema: `lsst.afw.table.Schema`
1405 Parameter catalog schema
1406 pars: `numpy.ndarray`
1407 FGCM parameters to put into parCat
1408 parSuperStarFlat: `numpy.array`
1409 FGCM superstar flat array to put into parCat
1410 lutFilterNameString: `str`
1411 Combined string of all the lutFilterNames
1412 fitBandString: `str`
1413 Combined string of all the fitBands
1414 notFitBandString: `str`
1415 Combined string of all the bands not used in the fit
1417 Returns
1418 -------
1419 parCat: `afwTable.BasicCatalog`
1420 Atmosphere and instrumental model parameter catalog for persistence
1421 """
1423 parCat = afwTable.BaseCatalog(parSchema)
1424 parCat.reserve(1)
1426 # The parameter catalog just has one row, with many columns for all the
1427 # atmosphere and instrument fit parameters
1428 rec = parCat.addNew()
1430 # info section
1431 rec['nCcd'] = parInfo['NCCD']
1432 rec['lutFilterNames'] = lutFilterNameString
1433 rec['fitBands'] = fitBandString
1434 rec['notFitBands'] = notFitBandString
1435 # note these are not currently supported here.
1436 rec['hasExternalPwv'] = 0
1437 rec['hasExternalTau'] = 0
1439 # parameter section
1441 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1443 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1444 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1445 'parQeSysIntercept', 'compQeSysSlope',
1446 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1447 'parFilterOffset', 'parFilterOffsetFitFlag',
1448 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1449 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1450 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1451 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1452 'compModelErrSkyPivot', 'compModelErrPars',
1453 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1454 'compSigmaCal',
1455 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1456 'compRetrievedTauNight']
1458 for scalarName in scalarNames:
1459 rec[scalarName] = pars[scalarName.upper()]
1461 for arrName in arrNames:
1462 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1464 # superstar section
1465 rec['superstarSize'][:] = parSuperStarFlat.shape
1466 rec['superstar'][:] = parSuperStarFlat.flatten()
1468 return parCat
1470 def _makeFlagStarSchema(self):
1471 """
1472 Make the flagged-stars schema
1474 Returns
1475 -------
1476 flagStarSchema: `lsst.afw.table.Schema`
1477 """
1479 flagStarSchema = afwTable.Schema()
1481 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1482 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1484 return flagStarSchema
1486 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1487 """
1488 Make the flagged star catalog for persistence
1490 Parameters
1491 ----------
1492 flagStarSchema: `lsst.afw.table.Schema`
1493 Flagged star schema
1494 flagStarStruct: `numpy.ndarray`
1495 Flagged star structure from fgcm
1497 Returns
1498 -------
1499 flagStarCat: `lsst.afw.table.BaseCatalog`
1500 Flagged star catalog for persistence
1501 """
1503 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1504 flagStarCat.reserve(flagStarStruct.size)
1505 for i in range(flagStarStruct.size):
1506 flagStarCat.addNew()
1508 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1509 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1511 return flagStarCat