Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44import lsst.afw.table as afwTable
46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47from .utilities import extractReferenceMags
48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50from .sedterms import SedboundarytermDict, SedtermDict
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57class FgcmFitCycleConfig(pexConfig.Config):
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc="Bands to run calibration",
62 dtype=str,
63 default=[],
64 )
65 fitFlag = pexConfig.ListField(
66 doc=("Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
70 dtype=int,
71 default=(0,),
72 optional=True,
73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
74 "It will be removed after v20. Use fitBands instead."),
75 )
76 fitBands = pexConfig.ListField(
77 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
78 "the atmosphere constrained from the 'fitBands' on the same night. "
79 "Must be a subset of `config.bands`"),
80 dtype=str,
81 default=[],
82 )
83 requiredFlag = pexConfig.ListField(
84 doc=("Flag for which bands are required for a star to be considered a calibration "
85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86 "be same length as config.bands, and matched band-by-band."),
87 dtype=int,
88 default=(0,),
89 optional=True,
90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
91 "It will be removed after v20. Use requiredBands instead."),
92 )
93 requiredBands = pexConfig.ListField(
94 doc=("Bands that are required for a star to be considered a calibration star. "
95 "Must be a subset of `config.bands`"),
96 dtype=str,
97 default=[],
98 )
99 filterMap = pexConfig.DictField(
100 doc="Mapping from 'filterName' to band.",
101 keytype=str,
102 itemtype=str,
103 default={},
104 )
105 doReferenceCalibration = pexConfig.Field(
106 doc="Use reference catalog as additional constraint on calibration",
107 dtype=bool,
108 default=True,
109 )
110 refStarSnMin = pexConfig.Field(
111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
112 dtype=float,
113 default=50.0,
114 )
115 refStarOutlierNSig = pexConfig.Field(
116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
118 dtype=float,
119 default=4.0,
120 )
121 applyRefStarColorCuts = pexConfig.Field(
122 doc="Apply color cuts to reference stars?",
123 dtype=bool,
124 default=True,
125 )
126 nCore = pexConfig.Field(
127 doc="Number of cores to use",
128 dtype=int,
129 default=4,
130 )
131 nStarPerRun = pexConfig.Field(
132 doc="Number of stars to run in each chunk",
133 dtype=int,
134 default=200000,
135 )
136 nExpPerRun = pexConfig.Field(
137 doc="Number of exposures to run in each chunk",
138 dtype=int,
139 default=1000,
140 )
141 reserveFraction = pexConfig.Field(
142 doc="Fraction of stars to reserve for testing",
143 dtype=float,
144 default=0.1,
145 )
146 freezeStdAtmosphere = pexConfig.Field(
147 doc="Freeze atmosphere parameters to standard (for testing)",
148 dtype=bool,
149 default=False,
150 )
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc="Precompute superstar flat for initial cycle",
153 dtype=bool,
154 default=False,
155 )
156 superStarSubCcd = pexConfig.Field(
157 doc="Compute superstar flat on sub-ccd scale",
158 dtype=bool,
159 default=True,
160 optional=True,
161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
162 "It will be removed after v20. Use superStarSubCcdDict instead."),
163 )
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166 "Must have one entry per band."),
167 keytype=str,
168 itemtype=bool,
169 default={},
170 )
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173 "Global default is first-order polynomials, and should be overridden "
174 "on a camera-by-camera basis depending on the ISR."),
175 dtype=int,
176 default=1,
177 )
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
180 "suppress high-order cross terms?"),
181 dtype=bool,
182 default=False,
183 )
184 superStarSigmaClip = pexConfig.Field(
185 doc="Number of sigma to clip outliers when selecting for superstar flats",
186 dtype=float,
187 default=5.0,
188 )
189 ccdGraySubCcd = pexConfig.Field(
190 doc="Compute CCD gray terms on sub-ccd scale",
191 dtype=bool,
192 default=False,
193 optional=True,
194 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
195 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
196 )
197 ccdGraySubCcdDict = pexConfig.DictField(
198 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
199 "('ccd gray') on a sub-ccd scale."),
200 keytype=str,
201 itemtype=bool,
202 default={},
203 )
204 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
206 dtype=int,
207 default=1,
208 )
209 ccdGraySubCcdTriangular = pexConfig.Field(
210 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
211 "suppress high-order cross terms?"),
212 dtype=bool,
213 default=True,
214 )
215 cycleNumber = pexConfig.Field(
216 doc=("FGCM fit cycle number. This is automatically incremented after each run "
217 "and stage of outlier rejection. See cookbook for details."),
218 dtype=int,
219 default=None,
220 )
221 isFinalCycle = pexConfig.Field(
222 doc=("Is this the final cycle of the fitting? Will automatically compute final "
223 "selection of stars and photometric exposures, and will output zeropoints "
224 "and standard stars for use in fgcmOutputProducts"),
225 dtype=bool,
226 default=False,
227 )
228 maxIterBeforeFinalCycle = pexConfig.Field(
229 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
230 "will always be 0 in the final cycle for cleanup and final selection."),
231 dtype=int,
232 default=50,
233 )
234 utBoundary = pexConfig.Field(
235 doc="Boundary (in UTC) from day-to-day",
236 dtype=float,
237 default=None,
238 )
239 washMjds = pexConfig.ListField(
240 doc="Mirror wash MJDs",
241 dtype=float,
242 default=(0.0,),
243 )
244 epochMjds = pexConfig.ListField(
245 doc="Epoch boundaries in MJD",
246 dtype=float,
247 default=(0.0,),
248 )
249 minObsPerBand = pexConfig.Field(
250 doc="Minimum good observations per band",
251 dtype=int,
252 default=2,
253 )
254 # TODO: When DM-16511 is done, it will be possible to get the
255 # telescope latitude directly from the camera.
256 latitude = pexConfig.Field(
257 doc="Observatory latitude",
258 dtype=float,
259 default=None,
260 )
261 pixelScale = pexConfig.Field(
262 doc="Pixel scale (arcsec/pixel) (temporary)",
263 dtype=float,
264 deprecated=("This field is no longer used, and has been deprecated by DM-16490. "
265 "It will be removed after v19."),
266 optional=True,
267 )
268 brightObsGrayMax = pexConfig.Field(
269 doc="Maximum gray extinction to be considered bright observation",
270 dtype=float,
271 default=0.15,
272 )
273 minStarPerCcd = pexConfig.Field(
274 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
275 "CCDs with fewer stars will have their calibration estimated from other "
276 "CCDs in the same visit, with zeropoint error increased accordingly."),
277 dtype=int,
278 default=5,
279 )
280 minCcdPerExp = pexConfig.Field(
281 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
282 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
283 dtype=int,
284 default=5,
285 )
286 maxCcdGrayErr = pexConfig.Field(
287 doc="Maximum error on CCD gray offset to be considered photometric",
288 dtype=float,
289 default=0.05,
290 )
291 minStarPerExp = pexConfig.Field(
292 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
293 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
294 dtype=int,
295 default=600,
296 )
297 minExpPerNight = pexConfig.Field(
298 doc="Minimum number of good exposures/visits to consider a partly photometric night",
299 dtype=int,
300 default=10,
301 )
302 expGrayInitialCut = pexConfig.Field(
303 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
304 "observations."),
305 dtype=float,
306 default=-0.25,
307 )
308 expGrayPhotometricCut = pexConfig.ListField(
309 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
310 "Must be same length as config.bands, and matched band-by-band."),
311 dtype=float,
312 default=(0.0,),
313 optional=True,
314 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
315 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
316 )
317 expGrayPhotometricCutDict = pexConfig.DictField(
318 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
319 "('gray term') for a visit to be considered photometric. Must have one "
320 "entry per band. Broad-band filters should be -0.05."),
321 keytype=str,
322 itemtype=float,
323 default={},
324 )
325 expGrayHighCut = pexConfig.ListField(
326 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
327 "Must be same length as config.bands, and matched band-by-band."),
328 dtype=float,
329 default=(0.0,),
330 optional=True,
331 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
332 "It will be removed after v20. Use expGrayHighCutDict instead."),
333 )
334 expGrayHighCutDict = pexConfig.DictField(
335 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
336 "('gray term') for a visit to be considered photometric. Must have one "
337 "entry per band. Broad-band filters should be 0.2."),
338 keytype=str,
339 itemtype=float,
340 default={},
341 )
342 expGrayRecoverCut = pexConfig.Field(
343 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
344 "Visits with more gray extinction will only get CCD zeropoints if there are "
345 "sufficient star observations (minStarPerCcd) on that CCD."),
346 dtype=float,
347 default=-1.0,
348 )
349 expVarGrayPhotometricCut = pexConfig.Field(
350 doc="Maximum exposure variance to be considered possibly photometric",
351 dtype=float,
352 default=0.0005,
353 optional=True,
354 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
355 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
356 )
357 expVarGrayPhotometricCutDict = pexConfig.DictField(
358 doc=("Per-band specification on maximum exposure variance to be considered possibly "
359 "photometric. Must have one entry per band. Broad-band filters should be "
360 "0.0005."),
361 keytype=str,
362 itemtype=float,
363 default={},
364 )
365 expGrayErrRecoverCut = pexConfig.Field(
366 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
367 "Visits with more gray variance will only get CCD zeropoints if there are "
368 "sufficient star observations (minStarPerCcd) on that CCD."),
369 dtype=float,
370 default=0.05,
371 )
372 aperCorrFitNBins = pexConfig.Field(
373 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
374 "no fit will be performed, and the config.aperCorrInputSlopes will be "
375 "used if available."),
376 dtype=int,
377 default=10,
378 )
379 aperCorrInputSlopes = pexConfig.ListField(
380 doc=("Aperture correction input slope parameters. These are used on the first "
381 "fit iteration, and aperture correction parameters will be updated from "
382 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
383 "when there is insufficient data to fit the parameters (e.g. tract mode). "
384 "If set, must be same length as config.bands, and matched band-by-band."),
385 dtype=float,
386 default=[],
387 optional=True,
388 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
389 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
390 )
391 aperCorrInputSlopeDict = pexConfig.DictField(
392 doc=("Per-band specification of aperture correction input slope parameters. These "
393 "are used on the first fit iteration, and aperture correction parameters will "
394 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
395 "to set this when there is insufficient data to fit the parameters (e.g. "
396 "tract mode)."),
397 keytype=str,
398 itemtype=float,
399 default={},
400 )
401 sedFudgeFactors = pexConfig.ListField(
402 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
403 "config.bands, and matched band-by-band."),
404 dtype=float,
405 default=(0,),
406 optional=True,
407 deprecated=("This field has been deprecated and will be removed after v20. "
408 "Please use sedSlopeTermMap and sedSlopeMap."),
409 )
410 sedboundaryterms = pexConfig.ConfigField(
411 doc="Mapping from bands to SED boundary term names used is sedterms.",
412 dtype=SedboundarytermDict,
413 )
414 sedterms = pexConfig.ConfigField(
415 doc="Mapping from terms to bands for fgcm linear SED approximations.",
416 dtype=SedtermDict,
417 )
418 sigFgcmMaxErr = pexConfig.Field(
419 doc="Maximum mag error for fitting sigma_FGCM",
420 dtype=float,
421 default=0.01,
422 )
423 sigFgcmMaxEGray = pexConfig.ListField(
424 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
425 "May be 1 element (same for all bands) or the same length as config.bands."),
426 dtype=float,
427 default=(0.05,),
428 optional=True,
429 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
430 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
431 )
432 sigFgcmMaxEGrayDict = pexConfig.DictField(
433 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
434 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
435 "should be 0.05."),
436 keytype=str,
437 itemtype=float,
438 default={},
439 )
440 ccdGrayMaxStarErr = pexConfig.Field(
441 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
442 "computation"),
443 dtype=float,
444 default=0.10,
445 )
446 approxThroughput = pexConfig.ListField(
447 doc=("Approximate overall throughput at start of calibration observations. "
448 "May be 1 element (same for all bands) or the same length as config.bands."),
449 dtype=float,
450 default=(1.0, ),
451 optional=True,
452 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
453 "It will be removed after v20. Use approxThroughputDict instead."),
454 )
455 approxThroughputDict = pexConfig.DictField(
456 doc=("Per-band specification of the approximate overall throughput at the start of "
457 "calibration observations. Must have one entry per band. Typically should "
458 "be 1.0."),
459 keytype=str,
460 itemtype=float,
461 default={},
462 )
463 sigmaCalRange = pexConfig.ListField(
464 doc="Allowed range for systematic error floor estimation",
465 dtype=float,
466 default=(0.001, 0.003),
467 )
468 sigmaCalFitPercentile = pexConfig.ListField(
469 doc="Magnitude percentile range to fit systematic error floor",
470 dtype=float,
471 default=(0.05, 0.15),
472 )
473 sigmaCalPlotPercentile = pexConfig.ListField(
474 doc="Magnitude percentile range to plot systematic error floor",
475 dtype=float,
476 default=(0.05, 0.95),
477 )
478 sigma0Phot = pexConfig.Field(
479 doc="Systematic error floor for all zeropoints",
480 dtype=float,
481 default=0.003,
482 )
483 mapLongitudeRef = pexConfig.Field(
484 doc="Reference longitude for plotting maps",
485 dtype=float,
486 default=0.0,
487 )
488 mapNSide = pexConfig.Field(
489 doc="Healpix nside for plotting maps",
490 dtype=int,
491 default=256,
492 )
493 outfileBase = pexConfig.Field(
494 doc="Filename start for plot output files",
495 dtype=str,
496 default=None,
497 )
498 starColorCuts = pexConfig.ListField(
499 doc="Encoded star-color cuts (to be cleaned up)",
500 dtype=str,
501 default=("NO_DATA",),
502 )
503 colorSplitIndices = pexConfig.ListField(
504 doc="Band indices to use to split stars by color",
505 dtype=int,
506 default=None,
507 optional=True,
508 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
509 "It will be removed after v20. Use colorSplitBands instead."),
510 )
511 colorSplitBands = pexConfig.ListField(
512 doc="Band names to use to split stars by color. Must have 2 entries.",
513 dtype=str,
514 length=2,
515 default=('g', 'i'),
516 )
517 modelMagErrors = pexConfig.Field(
518 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
519 dtype=bool,
520 default=True,
521 )
522 useQuadraticPwv = pexConfig.Field(
523 doc="Model PWV with a quadratic term for variation through the night?",
524 dtype=bool,
525 default=False,
526 )
527 instrumentParsPerBand = pexConfig.Field(
528 doc=("Model instrumental parameters per band? "
529 "Otherwise, instrumental parameters (QE changes with time) are "
530 "shared among all bands."),
531 dtype=bool,
532 default=False,
533 )
534 instrumentSlopeMinDeltaT = pexConfig.Field(
535 doc=("Minimum time change (in days) between observations to use in constraining "
536 "instrument slope."),
537 dtype=float,
538 default=20.0,
539 )
540 fitMirrorChromaticity = pexConfig.Field(
541 doc="Fit (intraband) mirror chromatic term?",
542 dtype=bool,
543 default=False,
544 )
545 coatingMjds = pexConfig.ListField(
546 doc="Mirror coating dates in MJD",
547 dtype=float,
548 default=(0.0,),
549 )
550 outputStandardsBeforeFinalCycle = pexConfig.Field(
551 doc="Output standard stars prior to final cycle? Used in debugging.",
552 dtype=bool,
553 default=False,
554 )
555 outputZeropointsBeforeFinalCycle = pexConfig.Field(
556 doc="Output standard stars prior to final cycle? Used in debugging.",
557 dtype=bool,
558 default=False,
559 )
560 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
561 doc=("Use star repeatability (instead of exposures) for computing photometric "
562 "cuts? Recommended for tract mode or bands with few exposures. "
563 "May be 1 element (same for all bands) or the same length as config.bands."),
564 dtype=bool,
565 default=(False,),
566 optional=True,
567 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
568 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
569 )
570 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
571 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
572 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
573 keytype=str,
574 itemtype=bool,
575 default={},
576 )
577 autoPhotometricCutNSig = pexConfig.Field(
578 doc=("Number of sigma for automatic computation of (low) photometric cut. "
579 "Cut is based on exposure gray width (per band), unless "
580 "useRepeatabilityForExpGrayCuts is set, in which case the star "
581 "repeatability is used (also per band)."),
582 dtype=float,
583 default=3.0,
584 )
585 autoHighCutNSig = pexConfig.Field(
586 doc=("Number of sigma for automatic computation of (high) outlier cut. "
587 "Cut is based on exposure gray width (per band), unless "
588 "useRepeatabilityForExpGrayCuts is set, in which case the star "
589 "repeatability is used (also per band)."),
590 dtype=float,
591 default=4.0,
592 )
593 quietMode = pexConfig.Field(
594 doc="Be less verbose with logging.",
595 dtype=bool,
596 default=False,
597 )
599 def setDefaults(self):
600 pass
602 def validate(self):
603 super().validate()
605 for band in self.fitBands:
606 if band not in self.bands:
607 msg = 'fitBand %s not in bands' % (band)
608 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
609 for band in self.requiredBands:
610 if band not in self.bands:
611 msg = 'requiredBand %s not in bands' % (band)
612 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
613 for band in self.colorSplitBands:
614 if band not in self.bands:
615 msg = 'colorSplitBand %s not in bands' % (band)
616 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
617 for band in self.bands:
618 if band not in self.superStarSubCcdDict:
619 msg = 'band %s not in superStarSubCcdDict' % (band)
620 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
621 self, msg)
622 if band not in self.ccdGraySubCcdDict:
623 msg = 'band %s not in ccdGraySubCcdDict' % (band)
624 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
625 self, msg)
626 if band not in self.expGrayPhotometricCutDict:
627 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
628 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
629 self, msg)
630 if band not in self.expGrayHighCutDict:
631 msg = 'band %s not in expGrayHighCutDict' % (band)
632 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
633 self, msg)
634 if band not in self.expVarGrayPhotometricCutDict:
635 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
636 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
637 self, msg)
638 if band not in self.sigFgcmMaxEGrayDict:
639 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
640 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
641 self, msg)
642 if band not in self.approxThroughputDict:
643 msg = 'band %s not in approxThroughputDict' % (band)
644 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
645 self, msg)
646 if band not in self.useRepeatabilityForExpGrayCutsDict:
647 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
648 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
649 self, msg)
652class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
653 """Subclass of TaskRunner for fgcmFitCycleTask
655 fgcmFitCycleTask.run() takes one argument, the butler, and uses
656 stars and visits previously extracted from dataRefs by
657 fgcmBuildStars.
658 This Runner does not perform any dataRef parallelization, but the FGCM
659 code called by the Task uses python multiprocessing (see the "ncores"
660 config option).
661 """
663 @staticmethod
664 def getTargetList(parsedCmd):
665 """
666 Return a list with one element, the butler.
667 """
668 return [parsedCmd.butler]
670 def __call__(self, butler):
671 """
672 Parameters
673 ----------
674 butler: `lsst.daf.persistence.Butler`
676 Returns
677 -------
678 exitStatus: `list` with `pipeBase.Struct`
679 exitStatus (0: success; 1: failure)
680 """
682 task = self.TaskClass(config=self.config, log=self.log)
684 exitStatus = 0
685 if self.doRaise:
686 task.runDataRef(butler)
687 else:
688 try:
689 task.runDataRef(butler)
690 except Exception as e:
691 exitStatus = 1
692 task.log.fatal("Failed: %s" % e)
693 if not isinstance(e, pipeBase.TaskError):
694 traceback.print_exc(file=sys.stderr)
696 task.writeMetadata(butler)
698 # The task does not return any results:
699 return [pipeBase.Struct(exitStatus=exitStatus)]
701 def run(self, parsedCmd):
702 """
703 Run the task, with no multiprocessing
705 Parameters
706 ----------
707 parsedCmd: ArgumentParser parsed command line
708 """
710 resultList = []
712 if self.precall(parsedCmd):
713 targetList = self.getTargetList(parsedCmd)
714 # make sure that we only get 1
715 resultList = self(targetList[0])
717 return resultList
720class FgcmFitCycleTask(pipeBase.CmdLineTask):
721 """
722 Run Single fit cycle for FGCM global calibration
723 """
725 ConfigClass = FgcmFitCycleConfig
726 RunnerClass = FgcmFitCycleRunner
727 _DefaultName = "fgcmFitCycle"
729 def __init__(self, butler=None, **kwargs):
730 """
731 Instantiate an fgcmFitCycle.
733 Parameters
734 ----------
735 butler : `lsst.daf.persistence.Butler`
736 """
738 pipeBase.CmdLineTask.__init__(self, **kwargs)
740 # no saving of metadata for now
741 def _getMetadataName(self):
742 return None
744 @pipeBase.timeMethod
745 def runDataRef(self, butler):
746 """
747 Run a single fit cycle for FGCM
749 Parameters
750 ----------
751 butler: `lsst.daf.persistence.Butler`
752 """
754 self._fgcmFitCycle(butler)
756 def writeConfig(self, butler, clobber=False, doBackup=True):
757 """Write the configuration used for processing the data, or check that an existing
758 one is equal to the new one if present. This is an override of the regular
759 version from pipe_base that knows about fgcmcycle.
761 Parameters
762 ----------
763 butler : `lsst.daf.persistence.Butler`
764 Data butler used to write the config. The config is written to dataset type
765 `CmdLineTask._getConfigName`.
766 clobber : `bool`, optional
767 A boolean flag that controls what happens if a config already has been saved:
768 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
769 - `False`: raise `TaskError` if this config does not match the existing config.
770 doBackup : `bool`, optional
771 Set to `True` to backup the config files if clobbering.
772 """
773 configName = self._getConfigName()
774 if configName is None:
775 return
776 if clobber:
777 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
778 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
779 # this may be subject to a race condition; see #2789
780 try:
781 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
782 except Exception as exc:
783 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
784 (configName, exc))
786 def logConfigMismatch(msg):
787 self.log.fatal("Comparing configuration: %s", msg)
789 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
790 raise pipeBase.TaskError(
791 ("Config does not match existing task config %r on disk; tasks configurations " +
792 "must be consistent within the same output repo (override with --clobber-config)") %
793 (configName,))
794 else:
795 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
797 def _fgcmFitCycle(self, butler):
798 """
799 Run the fit cycle
801 Parameters
802 ----------
803 butler: `lsst.daf.persistence.Butler`
804 """
806 self._checkDatasetsExist(butler)
808 # Set defaults on whether to output standards and zeropoints
809 self.maxIter = self.config.maxIterBeforeFinalCycle
810 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
811 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
812 self.resetFitParameters = True
814 if self.config.isFinalCycle:
815 # This is the final fit cycle, so we do not want to reset fit
816 # parameters, we want to run a final "clean-up" with 0 fit iterations,
817 # and we always want to output standards and zeropoints
818 self.maxIter = 0
819 self.outputStandards = True
820 self.outputZeropoints = True
821 self.resetFitParameters = False
823 camera = butler.get('camera')
824 configDict = makeConfigDict(self.config, self.log, camera,
825 self.maxIter, self.resetFitParameters,
826 self.outputZeropoints)
828 lutCat = butler.get('fgcmLookUpTable')
829 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
830 del lutCat
832 # next we need the exposure/visit information
834 # fgcmExpInfo = self._loadVisitCatalog(butler)
835 visitCat = butler.get('fgcmVisitCatalog')
836 fgcmExpInfo = translateVisitCatalog(visitCat)
837 del visitCat
839 # Use the first orientation.
840 # TODO: DM-21215 will generalize to arbitrary camera orientations
841 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
843 noFitsDict = {'lutIndex': lutIndexVals,
844 'lutStd': lutStd,
845 'expInfo': fgcmExpInfo,
846 'ccdOffsets': ccdOffsets}
848 # set up the fitter object
849 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
850 noFitsDict=noFitsDict, noOutput=True)
852 # create the parameter object
853 if (fgcmFitCycle.initialCycle):
854 # cycle = 0, initial cycle
855 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
856 fgcmLut,
857 fgcmExpInfo)
858 else:
859 inParInfo, inParams, inSuperStar = self._loadParameters(butler)
860 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
861 fgcmExpInfo,
862 inParInfo,
863 inParams,
864 inSuperStar)
866 lastCycle = configDict['cycleNumber'] - 1
868 # set up the stars...
869 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
871 starObs = butler.get('fgcmStarObservations')
872 starIds = butler.get('fgcmStarIds')
873 starIndices = butler.get('fgcmStarIndices')
875 # grab the flagged stars if available
876 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
877 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
878 flagId = flaggedStars['objId'][:]
879 flagFlag = flaggedStars['objFlag'][:]
880 else:
881 flagId = None
882 flagFlag = None
884 if self.config.doReferenceCalibration:
885 refStars = butler.get('fgcmReferenceStars')
887 refMag, refMagErr = extractReferenceMags(refStars,
888 self.config.bands,
889 self.config.filterMap)
890 refId = refStars['fgcm_id'][:]
891 else:
892 refId = None
893 refMag = None
894 refMagErr = None
896 # match star observations to visits
897 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
898 # actually be transferred into fgcm using the indexing below.
899 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
901 # The fgcmStars.loadStars method will copy all the star information into
902 # special shared memory objects that will not blow up the memory usage when
903 # used with python multiprocessing. Once all the numbers are copied,
904 # it is necessary to release all references to the objects that previously
905 # stored the data to ensure that the garbage collector can clear the memory,
906 # and ensure that this memory is not copied when multiprocessing kicks in.
908 # We determine the conversion from the native units (typically radians) to
909 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
910 # numpy arrays rather than Angles, which would we approximately 600x slower.
911 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
913 fgcmStars.loadStars(fgcmPars,
914 starObs['visit'][starIndices['obsIndex']],
915 starObs['ccd'][starIndices['obsIndex']],
916 starObs['ra'][starIndices['obsIndex']] * conv,
917 starObs['dec'][starIndices['obsIndex']] * conv,
918 starObs['instMag'][starIndices['obsIndex']],
919 starObs['instMagErr'][starIndices['obsIndex']],
920 fgcmExpInfo['FILTERNAME'][visitIndex],
921 starIds['fgcm_id'][:],
922 starIds['ra'][:],
923 starIds['dec'][:],
924 starIds['obsArrIndex'][:],
925 starIds['nObs'][:],
926 obsX=starObs['x'][starIndices['obsIndex']],
927 obsY=starObs['y'][starIndices['obsIndex']],
928 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
929 refID=refId,
930 refMag=refMag,
931 refMagErr=refMagErr,
932 flagID=flagId,
933 flagFlag=flagFlag,
934 computeNobs=True)
936 # Release all references to temporary objects holding star data (see above)
937 starObs = None
938 starIds = None
939 starIndices = None
940 flagId = None
941 flagFlag = None
942 flaggedStars = None
943 refStars = None
945 # and set the bits in the cycle object
946 fgcmFitCycle.setLUT(fgcmLut)
947 fgcmFitCycle.setStars(fgcmStars)
948 fgcmFitCycle.setPars(fgcmPars)
950 # finish the setup
951 fgcmFitCycle.finishSetup()
953 # and run
954 fgcmFitCycle.run()
956 ##################
957 # Persistance
958 ##################
960 self._persistFgcmDatasets(butler, fgcmFitCycle)
962 # Output the config for the next cycle
963 # We need to make a copy since the input one has been frozen
965 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
966 i, b in enumerate(self.config.bands)}
967 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
968 i, band in enumerate(self.config.bands)}
970 outConfig = copy.copy(self.config)
971 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
972 precomputeSuperStarInitialCycle=False,
973 freezeStdAtmosphere=False,
974 expGrayPhotometricCutDict=updatedPhotometricCutDict,
975 expGrayHighCutDict=updatedHighCutDict)
976 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
977 outConfig.cycleNumber)
978 outConfig.save(configFileName)
980 if self.config.isFinalCycle == 1:
981 # We are done, ready to output products
982 self.log.info("Everything is in place to run fgcmOutputProducts.py")
983 else:
984 self.log.info("Saved config for next cycle to %s" % (configFileName))
985 self.log.info("Be sure to look at:")
986 self.log.info(" config.expGrayPhotometricCut")
987 self.log.info(" config.expGrayHighCut")
988 self.log.info("If you are satisfied with the fit, please set:")
989 self.log.info(" config.isFinalCycle = True")
991 def _checkDatasetsExist(self, butler):
992 """
993 Check if necessary datasets exist to run fgcmFitCycle
995 Parameters
996 ----------
997 butler: `lsst.daf.persistence.Butler`
999 Raises
1000 ------
1001 RuntimeError
1002 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1003 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1004 If cycleNumber > 0, then also checks for fgcmFitParameters,
1005 fgcmFlaggedStars.
1006 """
1008 if not butler.datasetExists('fgcmVisitCatalog'):
1009 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1010 if not butler.datasetExists('fgcmStarObservations'):
1011 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1012 if not butler.datasetExists('fgcmStarIds'):
1013 raise RuntimeError("Could not find fgcmStarIds in repo!")
1014 if not butler.datasetExists('fgcmStarIndices'):
1015 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1016 if not butler.datasetExists('fgcmLookUpTable'):
1017 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1019 # Need additional datasets if we are not the initial cycle
1020 if (self.config.cycleNumber > 0):
1021 if not butler.datasetExists('fgcmFitParameters',
1022 fgcmcycle=self.config.cycleNumber-1):
1023 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1024 (self.config.cycleNumber-1))
1025 if not butler.datasetExists('fgcmFlaggedStars',
1026 fgcmcycle=self.config.cycleNumber-1):
1027 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1028 (self.config.cycleNumber-1))
1030 # And additional dataset if we want reference calibration
1031 if self.config.doReferenceCalibration:
1032 if not butler.datasetExists('fgcmReferenceStars'):
1033 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1034 "doReferenceCalibration is True.")
1036 def _loadParameters(self, butler):
1037 """
1038 Load FGCM parameters from a previous fit cycle
1040 Parameters
1041 ----------
1042 butler: `lsst.daf.persistence.Butler`
1044 Returns
1045 -------
1046 inParInfo: `numpy.ndarray`
1047 Numpy array parameter information formatted for input to fgcm
1048 inParameters: `numpy.ndarray`
1049 Numpy array parameter values formatted for input to fgcm
1050 inSuperStar: `numpy.array`
1051 Superstar flat formatted for input to fgcm
1052 """
1054 # note that we already checked that this is available
1055 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1057 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1058 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1059 parNotFitBands = np.array(parCat[0]['notFitBands'].split(','))
1061 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1062 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1063 (parLutFilterNames.size, )),
1064 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1065 ('NOTFITBANDS', parNotFitBands.dtype.str, (parNotFitBands.size, )),
1066 ('LNTAUUNIT', 'f8'),
1067 ('LNTAUSLOPEUNIT', 'f8'),
1068 ('ALPHAUNIT', 'f8'),
1069 ('LNPWVUNIT', 'f8'),
1070 ('LNPWVSLOPEUNIT', 'f8'),
1071 ('LNPWVQUADRATICUNIT', 'f8'),
1072 ('LNPWVGLOBALUNIT', 'f8'),
1073 ('O3UNIT', 'f8'),
1074 ('QESYSUNIT', 'f8'),
1075 ('FILTEROFFSETUNIT', 'f8'),
1076 ('HASEXTERNALPWV', 'i2'),
1077 ('HASEXTERNALTAU', 'i2')])
1078 inParInfo['NCCD'] = parCat['nCcd']
1079 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1080 inParInfo['FITBANDS'][:] = parFitBands
1081 inParInfo['NOTFITBANDS'][:] = parNotFitBands
1082 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1083 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1085 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1086 ('PARO3', 'f8', (parCat['parO3'].size, )),
1087 ('PARLNTAUINTERCEPT', 'f8',
1088 (parCat['parLnTauIntercept'].size, )),
1089 ('PARLNTAUSLOPE', 'f8',
1090 (parCat['parLnTauSlope'].size, )),
1091 ('PARLNPWVINTERCEPT', 'f8',
1092 (parCat['parLnPwvIntercept'].size, )),
1093 ('PARLNPWVSLOPE', 'f8',
1094 (parCat['parLnPwvSlope'].size, )),
1095 ('PARLNPWVQUADRATIC', 'f8',
1096 (parCat['parLnPwvQuadratic'].size, )),
1097 ('PARQESYSINTERCEPT', 'f8',
1098 (parCat['parQeSysIntercept'].size, )),
1099 ('COMPQESYSSLOPE', 'f8',
1100 (parCat['compQeSysSlope'].size, )),
1101 ('PARFILTEROFFSET', 'f8',
1102 (parCat['parFilterOffset'].size, )),
1103 ('PARFILTEROFFSETFITFLAG', 'i2',
1104 (parCat['parFilterOffsetFitFlag'].size, )),
1105 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1106 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1107 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1108 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1109 ('COMPABSTHROUGHPUT', 'f8',
1110 (parCat['compAbsThroughput'].size, )),
1111 ('COMPREFOFFSET', 'f8',
1112 (parCat['compRefOffset'].size, )),
1113 ('COMPREFSIGMA', 'f8',
1114 (parCat['compRefSigma'].size, )),
1115 ('COMPMIRRORCHROMATICITY', 'f8',
1116 (parCat['compMirrorChromaticity'].size, )),
1117 ('MIRRORCHROMATICITYPIVOT', 'f8',
1118 (parCat['mirrorChromaticityPivot'].size, )),
1119 ('COMPAPERCORRPIVOT', 'f8',
1120 (parCat['compAperCorrPivot'].size, )),
1121 ('COMPAPERCORRSLOPE', 'f8',
1122 (parCat['compAperCorrSlope'].size, )),
1123 ('COMPAPERCORRSLOPEERR', 'f8',
1124 (parCat['compAperCorrSlopeErr'].size, )),
1125 ('COMPAPERCORRRANGE', 'f8',
1126 (parCat['compAperCorrRange'].size, )),
1127 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1128 (parCat['compModelErrExptimePivot'].size, )),
1129 ('COMPMODELERRFWHMPIVOT', 'f8',
1130 (parCat['compModelErrFwhmPivot'].size, )),
1131 ('COMPMODELERRSKYPIVOT', 'f8',
1132 (parCat['compModelErrSkyPivot'].size, )),
1133 ('COMPMODELERRPARS', 'f8',
1134 (parCat['compModelErrPars'].size, )),
1135 ('COMPEXPGRAY', 'f8',
1136 (parCat['compExpGray'].size, )),
1137 ('COMPVARGRAY', 'f8',
1138 (parCat['compVarGray'].size, )),
1139 ('COMPNGOODSTARPEREXP', 'i4',
1140 (parCat['compNGoodStarPerExp'].size, )),
1141 ('COMPSIGFGCM', 'f8',
1142 (parCat['compSigFgcm'].size, )),
1143 ('COMPSIGMACAL', 'f8',
1144 (parCat['compSigmaCal'].size, )),
1145 ('COMPRETRIEVEDLNPWV', 'f8',
1146 (parCat['compRetrievedLnPwv'].size, )),
1147 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1148 (parCat['compRetrievedLnPwvRaw'].size, )),
1149 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1150 (parCat['compRetrievedLnPwvFlag'].size, )),
1151 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1152 (parCat['compRetrievedTauNight'].size, ))])
1154 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1155 inParams['PARO3'][:] = parCat['parO3'][0, :]
1156 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1157 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1158 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1159 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1160 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1161 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1162 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1163 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1164 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1165 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1166 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1167 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1168 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1169 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1170 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1171 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1172 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1173 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1174 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1175 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1176 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1177 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1178 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1179 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1180 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1181 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1182 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1183 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1184 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1185 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1186 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1187 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1188 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1189 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1191 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1192 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1194 return (inParInfo, inParams, inSuperStar)
1196 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1197 """
1198 Persist FGCM datasets through the butler.
1200 Parameters
1201 ----------
1202 butler: `lsst.daf.persistence.Butler`
1203 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1204 Fgcm Fit cycle object
1205 """
1207 # Save the parameters
1208 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1210 parSchema = afwTable.Schema()
1212 comma = ','
1213 lutFilterNameString = comma.join([n.decode('utf-8')
1214 for n in parInfo['LUTFILTERNAMES'][0]])
1215 fitBandString = comma.join([n.decode('utf-8')
1216 for n in parInfo['FITBANDS'][0]])
1217 notFitBandString = comma.join([n.decode('utf-8')
1218 for n in parInfo['NOTFITBANDS'][0]])
1220 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1221 lutFilterNameString, fitBandString, notFitBandString)
1222 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1223 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1224 lutFilterNameString, fitBandString, notFitBandString)
1226 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1228 # Save the indices of the flagged stars
1229 # (stars that have been (a) reserved from the fit for testing and
1230 # (b) bad stars that have failed quality checks.)
1231 flagStarSchema = self._makeFlagStarSchema()
1232 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1233 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1235 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1237 # Save the zeropoint information and atmospheres only if desired
1238 if self.outputZeropoints:
1239 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1240 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1242 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1243 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1245 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1247 # Save atmosphere values
1248 # These are generated by the same code that generates zeropoints
1249 atmSchema = makeAtmSchema()
1250 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1252 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1254 # Save the standard stars (if configured)
1255 if self.outputStandards:
1256 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1257 stdSchema = makeStdSchema(len(goodBands))
1258 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1260 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1262 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1263 lutFilterNameString, fitBandString, notFitBandString):
1264 """
1265 Make the parameter persistence schema
1267 Parameters
1268 ----------
1269 parInfo: `numpy.ndarray`
1270 Parameter information returned by fgcm
1271 pars: `numpy.ndarray`
1272 Parameter values returned by fgcm
1273 parSuperStarFlat: `numpy.array`
1274 Superstar flat values returned by fgcm
1275 lutFilterNameString: `str`
1276 Combined string of all the lutFilterNames
1277 fitBandString: `str`
1278 Combined string of all the fitBands
1279 notFitBandString: `str`
1280 Combined string of all the bands not used in the fit
1282 Returns
1283 -------
1284 parSchema: `afwTable.schema`
1285 """
1287 parSchema = afwTable.Schema()
1289 # parameter info section
1290 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1291 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1292 size=len(lutFilterNameString))
1293 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1294 size=len(fitBandString))
1295 parSchema.addField('notFitBands', type=str, doc='Bands that were not fit',
1296 size=len(notFitBandString))
1297 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1298 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1299 doc='Step units for ln(AOD) slope')
1300 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1301 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1302 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1303 doc='Step units for ln(pwv) slope')
1304 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1305 doc='Step units for ln(pwv) quadratic term')
1306 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1307 doc='Step units for global ln(pwv) parameters')
1308 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1309 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1310 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1311 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1312 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1314 # parameter section
1315 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1316 size=pars['PARALPHA'].size)
1317 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1318 size=pars['PARO3'].size)
1319 parSchema.addField('parLnTauIntercept', type='ArrayD',
1320 doc='ln(Tau) intercept parameter vector',
1321 size=pars['PARLNTAUINTERCEPT'].size)
1322 parSchema.addField('parLnTauSlope', type='ArrayD',
1323 doc='ln(Tau) slope parameter vector',
1324 size=pars['PARLNTAUSLOPE'].size)
1325 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1326 size=pars['PARLNPWVINTERCEPT'].size)
1327 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1328 size=pars['PARLNPWVSLOPE'].size)
1329 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1330 size=pars['PARLNPWVQUADRATIC'].size)
1331 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1332 size=pars['PARQESYSINTERCEPT'].size)
1333 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1334 size=pars[0]['COMPQESYSSLOPE'].size)
1335 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1336 size=pars['PARFILTEROFFSET'].size)
1337 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1338 size=pars['PARFILTEROFFSETFITFLAG'].size)
1339 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1340 doc='Global scale for retrieved ln(pwv)')
1341 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1342 doc='Global offset for retrieved ln(pwv)')
1343 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1344 doc='Nightly offset for retrieved ln(pwv)',
1345 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1346 parSchema.addField('compAbsThroughput', type='ArrayD',
1347 doc='Absolute throughput (relative to transmission curves)',
1348 size=pars['COMPABSTHROUGHPUT'].size)
1349 parSchema.addField('compRefOffset', type='ArrayD',
1350 doc='Offset between reference stars and calibrated stars',
1351 size=pars['COMPREFOFFSET'].size)
1352 parSchema.addField('compRefSigma', type='ArrayD',
1353 doc='Width of reference star/calibrated star distribution',
1354 size=pars['COMPREFSIGMA'].size)
1355 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1356 doc='Computed mirror chromaticity terms',
1357 size=pars['COMPMIRRORCHROMATICITY'].size)
1358 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1359 doc='Mirror chromaticity pivot mjd',
1360 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1361 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1362 size=pars['COMPAPERCORRPIVOT'].size)
1363 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1364 size=pars['COMPAPERCORRSLOPE'].size)
1365 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1366 size=pars['COMPAPERCORRSLOPEERR'].size)
1367 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1368 size=pars['COMPAPERCORRRANGE'].size)
1369 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1370 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1371 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1372 size=pars['COMPMODELERRFWHMPIVOT'].size)
1373 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1374 size=pars['COMPMODELERRSKYPIVOT'].size)
1375 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1376 size=pars['COMPMODELERRPARS'].size)
1377 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1378 size=pars['COMPEXPGRAY'].size)
1379 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1380 size=pars['COMPVARGRAY'].size)
1381 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1382 doc='Computed number of good stars per exposure',
1383 size=pars['COMPNGOODSTARPEREXP'].size)
1384 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1385 size=pars['COMPSIGFGCM'].size)
1386 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1387 size=pars['COMPSIGMACAL'].size)
1388 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1389 size=pars['COMPRETRIEVEDLNPWV'].size)
1390 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1391 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1392 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1393 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1394 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1395 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1396 # superstarflat section
1397 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1398 size=4)
1399 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1400 size=parSuperStarFlat.size)
1402 return parSchema
1404 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1405 lutFilterNameString, fitBandString, notFitBandString):
1406 """
1407 Make the FGCM parameter catalog for persistence
1409 Parameters
1410 ----------
1411 parSchema: `lsst.afw.table.Schema`
1412 Parameter catalog schema
1413 pars: `numpy.ndarray`
1414 FGCM parameters to put into parCat
1415 parSuperStarFlat: `numpy.array`
1416 FGCM superstar flat array to put into parCat
1417 lutFilterNameString: `str`
1418 Combined string of all the lutFilterNames
1419 fitBandString: `str`
1420 Combined string of all the fitBands
1421 notFitBandString: `str`
1422 Combined string of all the bands not used in the fit
1424 Returns
1425 -------
1426 parCat: `afwTable.BasicCatalog`
1427 Atmosphere and instrumental model parameter catalog for persistence
1428 """
1430 parCat = afwTable.BaseCatalog(parSchema)
1431 parCat.reserve(1)
1433 # The parameter catalog just has one row, with many columns for all the
1434 # atmosphere and instrument fit parameters
1435 rec = parCat.addNew()
1437 # info section
1438 rec['nCcd'] = parInfo['NCCD']
1439 rec['lutFilterNames'] = lutFilterNameString
1440 rec['fitBands'] = fitBandString
1441 rec['notFitBands'] = notFitBandString
1442 # note these are not currently supported here.
1443 rec['hasExternalPwv'] = 0
1444 rec['hasExternalTau'] = 0
1446 # parameter section
1448 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1450 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1451 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1452 'parQeSysIntercept', 'compQeSysSlope',
1453 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1454 'parFilterOffset', 'parFilterOffsetFitFlag',
1455 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1456 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1457 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1458 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1459 'compModelErrSkyPivot', 'compModelErrPars',
1460 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1461 'compSigmaCal',
1462 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1463 'compRetrievedTauNight']
1465 for scalarName in scalarNames:
1466 rec[scalarName] = pars[scalarName.upper()]
1468 for arrName in arrNames:
1469 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1471 # superstar section
1472 rec['superstarSize'][:] = parSuperStarFlat.shape
1473 rec['superstar'][:] = parSuperStarFlat.flatten()
1475 return parCat
1477 def _makeFlagStarSchema(self):
1478 """
1479 Make the flagged-stars schema
1481 Returns
1482 -------
1483 flagStarSchema: `lsst.afw.table.Schema`
1484 """
1486 flagStarSchema = afwTable.Schema()
1488 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1489 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1491 return flagStarSchema
1493 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1494 """
1495 Make the flagged star catalog for persistence
1497 Parameters
1498 ----------
1499 flagStarSchema: `lsst.afw.table.Schema`
1500 Flagged star schema
1501 flagStarStruct: `numpy.ndarray`
1502 Flagged star structure from fgcm
1504 Returns
1505 -------
1506 flagStarCat: `lsst.afw.table.BaseCatalog`
1507 Flagged star catalog for persistence
1508 """
1510 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1511 flagStarCat.reserve(flagStarStruct.size)
1512 for i in range(flagStarStruct.size):
1513 flagStarCat.addNew()
1515 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1516 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1518 return flagStarCat