Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44import lsst.afw.table as afwTable
46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47from .utilities import extractReferenceMags
48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50from .sedterms import SedboundarytermDict, SedtermDict
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57class FgcmFitCycleConfig(pexConfig.Config):
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc="Bands to run calibration",
62 dtype=str,
63 default=[],
64 )
65 fitFlag = pexConfig.ListField(
66 doc=("Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
70 dtype=int,
71 default=(0,),
72 optional=True,
73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
74 "It will be removed after v20. Use fitBands instead."),
75 )
76 fitBands = pexConfig.ListField(
77 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
78 "the atmosphere constrained from the 'fitBands' on the same night. "
79 "Must be a subset of `config.bands`"),
80 dtype=str,
81 default=[],
82 )
83 requiredFlag = pexConfig.ListField(
84 doc=("Flag for which bands are required for a star to be considered a calibration "
85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86 "be same length as config.bands, and matched band-by-band."),
87 dtype=int,
88 default=(0,),
89 optional=True,
90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
91 "It will be removed after v20. Use requiredBands instead."),
92 )
93 requiredBands = pexConfig.ListField(
94 doc=("Bands that are required for a star to be considered a calibration star. "
95 "Must be a subset of `config.bands`"),
96 dtype=str,
97 default=[],
98 )
99 filterMap = pexConfig.DictField(
100 doc="Mapping from 'filterName' to band.",
101 keytype=str,
102 itemtype=str,
103 default={},
104 )
105 doReferenceCalibration = pexConfig.Field(
106 doc="Use reference catalog as additional constraint on calibration",
107 dtype=bool,
108 default=True,
109 )
110 refStarSnMin = pexConfig.Field(
111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
112 dtype=float,
113 default=50.0,
114 )
115 refStarOutlierNSig = pexConfig.Field(
116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
118 dtype=float,
119 default=4.0,
120 )
121 applyRefStarColorCuts = pexConfig.Field(
122 doc="Apply color cuts to reference stars?",
123 dtype=bool,
124 default=True,
125 )
126 nCore = pexConfig.Field(
127 doc="Number of cores to use",
128 dtype=int,
129 default=4,
130 )
131 nStarPerRun = pexConfig.Field(
132 doc="Number of stars to run in each chunk",
133 dtype=int,
134 default=200000,
135 )
136 nExpPerRun = pexConfig.Field(
137 doc="Number of exposures to run in each chunk",
138 dtype=int,
139 default=1000,
140 )
141 reserveFraction = pexConfig.Field(
142 doc="Fraction of stars to reserve for testing",
143 dtype=float,
144 default=0.1,
145 )
146 freezeStdAtmosphere = pexConfig.Field(
147 doc="Freeze atmosphere parameters to standard (for testing)",
148 dtype=bool,
149 default=False,
150 )
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc="Precompute superstar flat for initial cycle",
153 dtype=bool,
154 default=False,
155 )
156 superStarSubCcd = pexConfig.Field(
157 doc="Compute superstar flat on sub-ccd scale",
158 dtype=bool,
159 default=True,
160 optional=True,
161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
162 "It will be removed after v20. Use superStarSubCcdDict instead."),
163 )
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166 "Must have one entry per band."),
167 keytype=str,
168 itemtype=bool,
169 default={},
170 )
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173 "Global default is first-order polynomials, and should be overridden "
174 "on a camera-by-camera basis depending on the ISR."),
175 dtype=int,
176 default=1,
177 )
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
180 "suppress high-order cross terms?"),
181 dtype=bool,
182 default=False,
183 )
184 superStarSigmaClip = pexConfig.Field(
185 doc="Number of sigma to clip outliers when selecting for superstar flats",
186 dtype=float,
187 default=5.0,
188 )
189 focalPlaneSigmaClip = pexConfig.Field(
190 doc="Number of sigma to clip outliers per focal-plane.",
191 dtype=float,
192 default=4.0,
193 )
194 ccdGraySubCcd = pexConfig.Field(
195 doc="Compute CCD gray terms on sub-ccd scale",
196 dtype=bool,
197 default=False,
198 optional=True,
199 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
200 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
201 )
202 ccdGraySubCcdDict = pexConfig.DictField(
203 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
204 "('ccd gray') on a sub-ccd scale."),
205 keytype=str,
206 itemtype=bool,
207 default={},
208 )
209 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
210 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
211 dtype=int,
212 default=1,
213 )
214 ccdGraySubCcdTriangular = pexConfig.Field(
215 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
216 "suppress high-order cross terms?"),
217 dtype=bool,
218 default=True,
219 )
220 ccdGrayFocalPlaneDict = pexConfig.DictField(
221 doc=("Per-band specification on whether to compute focal-plane residual "
222 "('ccd gray') corrections."),
223 keytype=str,
224 itemtype=bool,
225 default={},
226 )
227 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
228 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
229 "gray corrections. If there are fewer good CCDs then the gray "
230 "correction is computed per-ccd."),
231 dtype=int,
232 default=1,
233 )
234 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
235 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
236 dtype=int,
237 default=3,
238 )
239 cycleNumber = pexConfig.Field(
240 doc=("FGCM fit cycle number. This is automatically incremented after each run "
241 "and stage of outlier rejection. See cookbook for details."),
242 dtype=int,
243 default=None,
244 )
245 isFinalCycle = pexConfig.Field(
246 doc=("Is this the final cycle of the fitting? Will automatically compute final "
247 "selection of stars and photometric exposures, and will output zeropoints "
248 "and standard stars for use in fgcmOutputProducts"),
249 dtype=bool,
250 default=False,
251 )
252 maxIterBeforeFinalCycle = pexConfig.Field(
253 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
254 "will always be 0 in the final cycle for cleanup and final selection."),
255 dtype=int,
256 default=50,
257 )
258 deltaMagBkgOffsetPercentile = pexConfig.Field(
259 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
260 "offset from local background subtraction."),
261 dtype=float,
262 default=0.25,
263 )
264 deltaMagBkgPerCcd = pexConfig.Field(
265 doc=("Compute net offset from local background subtraction per-ccd? "
266 "Otherwise, use computation per visit."),
267 dtype=bool,
268 default=False,
269 )
270 utBoundary = pexConfig.Field(
271 doc="Boundary (in UTC) from day-to-day",
272 dtype=float,
273 default=None,
274 )
275 washMjds = pexConfig.ListField(
276 doc="Mirror wash MJDs",
277 dtype=float,
278 default=(0.0,),
279 )
280 epochMjds = pexConfig.ListField(
281 doc="Epoch boundaries in MJD",
282 dtype=float,
283 default=(0.0,),
284 )
285 minObsPerBand = pexConfig.Field(
286 doc="Minimum good observations per band",
287 dtype=int,
288 default=2,
289 )
290 # TODO: When DM-16511 is done, it will be possible to get the
291 # telescope latitude directly from the camera.
292 latitude = pexConfig.Field(
293 doc="Observatory latitude",
294 dtype=float,
295 default=None,
296 )
297 brightObsGrayMax = pexConfig.Field(
298 doc="Maximum gray extinction to be considered bright observation",
299 dtype=float,
300 default=0.15,
301 )
302 minStarPerCcd = pexConfig.Field(
303 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
304 "CCDs with fewer stars will have their calibration estimated from other "
305 "CCDs in the same visit, with zeropoint error increased accordingly."),
306 dtype=int,
307 default=5,
308 )
309 minCcdPerExp = pexConfig.Field(
310 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
311 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
312 dtype=int,
313 default=5,
314 )
315 maxCcdGrayErr = pexConfig.Field(
316 doc="Maximum error on CCD gray offset to be considered photometric",
317 dtype=float,
318 default=0.05,
319 )
320 minStarPerExp = pexConfig.Field(
321 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
322 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
323 dtype=int,
324 default=600,
325 )
326 minExpPerNight = pexConfig.Field(
327 doc="Minimum number of good exposures/visits to consider a partly photometric night",
328 dtype=int,
329 default=10,
330 )
331 expGrayInitialCut = pexConfig.Field(
332 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
333 "observations."),
334 dtype=float,
335 default=-0.25,
336 )
337 expGrayPhotometricCut = pexConfig.ListField(
338 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
339 "Must be same length as config.bands, and matched band-by-band."),
340 dtype=float,
341 default=(0.0,),
342 optional=True,
343 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
344 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
345 )
346 expGrayPhotometricCutDict = pexConfig.DictField(
347 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
348 "('gray term') for a visit to be considered photometric. Must have one "
349 "entry per band. Broad-band filters should be -0.05."),
350 keytype=str,
351 itemtype=float,
352 default={},
353 )
354 expGrayHighCut = pexConfig.ListField(
355 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
356 "Must be same length as config.bands, and matched band-by-band."),
357 dtype=float,
358 default=(0.0,),
359 optional=True,
360 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
361 "It will be removed after v20. Use expGrayHighCutDict instead."),
362 )
363 expGrayHighCutDict = pexConfig.DictField(
364 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
365 "('gray term') for a visit to be considered photometric. Must have one "
366 "entry per band. Broad-band filters should be 0.2."),
367 keytype=str,
368 itemtype=float,
369 default={},
370 )
371 expGrayRecoverCut = pexConfig.Field(
372 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
373 "Visits with more gray extinction will only get CCD zeropoints if there are "
374 "sufficient star observations (minStarPerCcd) on that CCD."),
375 dtype=float,
376 default=-1.0,
377 )
378 expVarGrayPhotometricCut = pexConfig.Field(
379 doc="Maximum exposure variance to be considered possibly photometric",
380 dtype=float,
381 default=0.0005,
382 optional=True,
383 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
384 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
385 )
386 expVarGrayPhotometricCutDict = pexConfig.DictField(
387 doc=("Per-band specification on maximum exposure variance to be considered possibly "
388 "photometric. Must have one entry per band. Broad-band filters should be "
389 "0.0005."),
390 keytype=str,
391 itemtype=float,
392 default={},
393 )
394 expGrayErrRecoverCut = pexConfig.Field(
395 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
396 "Visits with more gray variance will only get CCD zeropoints if there are "
397 "sufficient star observations (minStarPerCcd) on that CCD."),
398 dtype=float,
399 default=0.05,
400 )
401 aperCorrFitNBins = pexConfig.Field(
402 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
403 "no fit will be performed, and the config.aperCorrInputSlopes will be "
404 "used if available."),
405 dtype=int,
406 default=10,
407 )
408 aperCorrInputSlopes = pexConfig.ListField(
409 doc=("Aperture correction input slope parameters. These are used on the first "
410 "fit iteration, and aperture correction parameters will be updated from "
411 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
412 "when there is insufficient data to fit the parameters (e.g. tract mode). "
413 "If set, must be same length as config.bands, and matched band-by-band."),
414 dtype=float,
415 default=[],
416 optional=True,
417 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
418 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
419 )
420 aperCorrInputSlopeDict = pexConfig.DictField(
421 doc=("Per-band specification of aperture correction input slope parameters. These "
422 "are used on the first fit iteration, and aperture correction parameters will "
423 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
424 "to set this when there is insufficient data to fit the parameters (e.g. "
425 "tract mode)."),
426 keytype=str,
427 itemtype=float,
428 default={},
429 )
430 sedFudgeFactors = pexConfig.ListField(
431 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
432 "config.bands, and matched band-by-band."),
433 dtype=float,
434 default=(0,),
435 optional=True,
436 deprecated=("This field has been deprecated and will be removed after v20. "
437 "Please use sedSlopeTermMap and sedSlopeMap."),
438 )
439 sedboundaryterms = pexConfig.ConfigField(
440 doc="Mapping from bands to SED boundary term names used is sedterms.",
441 dtype=SedboundarytermDict,
442 )
443 sedterms = pexConfig.ConfigField(
444 doc="Mapping from terms to bands for fgcm linear SED approximations.",
445 dtype=SedtermDict,
446 )
447 sigFgcmMaxErr = pexConfig.Field(
448 doc="Maximum mag error for fitting sigma_FGCM",
449 dtype=float,
450 default=0.01,
451 )
452 sigFgcmMaxEGray = pexConfig.ListField(
453 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
454 "May be 1 element (same for all bands) or the same length as config.bands."),
455 dtype=float,
456 default=(0.05,),
457 optional=True,
458 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
459 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
460 )
461 sigFgcmMaxEGrayDict = pexConfig.DictField(
462 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
463 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
464 "should be 0.05."),
465 keytype=str,
466 itemtype=float,
467 default={},
468 )
469 ccdGrayMaxStarErr = pexConfig.Field(
470 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
471 "computation"),
472 dtype=float,
473 default=0.10,
474 )
475 approxThroughput = pexConfig.ListField(
476 doc=("Approximate overall throughput at start of calibration observations. "
477 "May be 1 element (same for all bands) or the same length as config.bands."),
478 dtype=float,
479 default=(1.0, ),
480 optional=True,
481 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
482 "It will be removed after v20. Use approxThroughputDict instead."),
483 )
484 approxThroughputDict = pexConfig.DictField(
485 doc=("Per-band specification of the approximate overall throughput at the start of "
486 "calibration observations. Must have one entry per band. Typically should "
487 "be 1.0."),
488 keytype=str,
489 itemtype=float,
490 default={},
491 )
492 sigmaCalRange = pexConfig.ListField(
493 doc="Allowed range for systematic error floor estimation",
494 dtype=float,
495 default=(0.001, 0.003),
496 )
497 sigmaCalFitPercentile = pexConfig.ListField(
498 doc="Magnitude percentile range to fit systematic error floor",
499 dtype=float,
500 default=(0.05, 0.15),
501 )
502 sigmaCalPlotPercentile = pexConfig.ListField(
503 doc="Magnitude percentile range to plot systematic error floor",
504 dtype=float,
505 default=(0.05, 0.95),
506 )
507 sigma0Phot = pexConfig.Field(
508 doc="Systematic error floor for all zeropoints",
509 dtype=float,
510 default=0.003,
511 )
512 mapLongitudeRef = pexConfig.Field(
513 doc="Reference longitude for plotting maps",
514 dtype=float,
515 default=0.0,
516 )
517 mapNSide = pexConfig.Field(
518 doc="Healpix nside for plotting maps",
519 dtype=int,
520 default=256,
521 )
522 outfileBase = pexConfig.Field(
523 doc="Filename start for plot output files",
524 dtype=str,
525 default=None,
526 )
527 starColorCuts = pexConfig.ListField(
528 doc="Encoded star-color cuts (to be cleaned up)",
529 dtype=str,
530 default=("NO_DATA",),
531 )
532 colorSplitIndices = pexConfig.ListField(
533 doc="Band indices to use to split stars by color",
534 dtype=int,
535 default=None,
536 optional=True,
537 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
538 "It will be removed after v20. Use colorSplitBands instead."),
539 )
540 colorSplitBands = pexConfig.ListField(
541 doc="Band names to use to split stars by color. Must have 2 entries.",
542 dtype=str,
543 length=2,
544 default=('g', 'i'),
545 )
546 modelMagErrors = pexConfig.Field(
547 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
548 dtype=bool,
549 default=True,
550 )
551 useQuadraticPwv = pexConfig.Field(
552 doc="Model PWV with a quadratic term for variation through the night?",
553 dtype=bool,
554 default=False,
555 )
556 instrumentParsPerBand = pexConfig.Field(
557 doc=("Model instrumental parameters per band? "
558 "Otherwise, instrumental parameters (QE changes with time) are "
559 "shared among all bands."),
560 dtype=bool,
561 default=False,
562 )
563 instrumentSlopeMinDeltaT = pexConfig.Field(
564 doc=("Minimum time change (in days) between observations to use in constraining "
565 "instrument slope."),
566 dtype=float,
567 default=20.0,
568 )
569 fitMirrorChromaticity = pexConfig.Field(
570 doc="Fit (intraband) mirror chromatic term?",
571 dtype=bool,
572 default=False,
573 )
574 coatingMjds = pexConfig.ListField(
575 doc="Mirror coating dates in MJD",
576 dtype=float,
577 default=(0.0,),
578 )
579 outputStandardsBeforeFinalCycle = pexConfig.Field(
580 doc="Output standard stars prior to final cycle? Used in debugging.",
581 dtype=bool,
582 default=False,
583 )
584 outputZeropointsBeforeFinalCycle = pexConfig.Field(
585 doc="Output standard stars prior to final cycle? Used in debugging.",
586 dtype=bool,
587 default=False,
588 )
589 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
590 doc=("Use star repeatability (instead of exposures) for computing photometric "
591 "cuts? Recommended for tract mode or bands with few exposures. "
592 "May be 1 element (same for all bands) or the same length as config.bands."),
593 dtype=bool,
594 default=(False,),
595 optional=True,
596 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
597 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
598 )
599 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
600 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
601 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
602 keytype=str,
603 itemtype=bool,
604 default={},
605 )
606 autoPhotometricCutNSig = pexConfig.Field(
607 doc=("Number of sigma for automatic computation of (low) photometric cut. "
608 "Cut is based on exposure gray width (per band), unless "
609 "useRepeatabilityForExpGrayCuts is set, in which case the star "
610 "repeatability is used (also per band)."),
611 dtype=float,
612 default=3.0,
613 )
614 autoHighCutNSig = pexConfig.Field(
615 doc=("Number of sigma for automatic computation of (high) outlier cut. "
616 "Cut is based on exposure gray width (per band), unless "
617 "useRepeatabilityForExpGrayCuts is set, in which case the star "
618 "repeatability is used (also per band)."),
619 dtype=float,
620 default=4.0,
621 )
622 quietMode = pexConfig.Field(
623 doc="Be less verbose with logging.",
624 dtype=bool,
625 default=False,
626 )
627 randomSeed = pexConfig.Field(
628 doc="Random seed for fgcm for consistency in tests.",
629 dtype=int,
630 default=None,
631 optional=True,
632 )
634 def setDefaults(self):
635 pass
637 def validate(self):
638 super().validate()
640 for band in self.fitBands:
641 if band not in self.bands:
642 msg = 'fitBand %s not in bands' % (band)
643 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
644 for band in self.requiredBands:
645 if band not in self.bands:
646 msg = 'requiredBand %s not in bands' % (band)
647 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
648 for band in self.colorSplitBands:
649 if band not in self.bands:
650 msg = 'colorSplitBand %s not in bands' % (band)
651 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
652 for band in self.bands:
653 if band not in self.superStarSubCcdDict:
654 msg = 'band %s not in superStarSubCcdDict' % (band)
655 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
656 self, msg)
657 if band not in self.ccdGraySubCcdDict:
658 msg = 'band %s not in ccdGraySubCcdDict' % (band)
659 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
660 self, msg)
661 if band not in self.expGrayPhotometricCutDict:
662 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
663 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
664 self, msg)
665 if band not in self.expGrayHighCutDict:
666 msg = 'band %s not in expGrayHighCutDict' % (band)
667 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
668 self, msg)
669 if band not in self.expVarGrayPhotometricCutDict:
670 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
671 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
672 self, msg)
673 if band not in self.sigFgcmMaxEGrayDict:
674 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
675 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
676 self, msg)
677 if band not in self.approxThroughputDict:
678 msg = 'band %s not in approxThroughputDict' % (band)
679 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
680 self, msg)
681 if band not in self.useRepeatabilityForExpGrayCutsDict:
682 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
683 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
684 self, msg)
687class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
688 """Subclass of TaskRunner for fgcmFitCycleTask
690 fgcmFitCycleTask.run() takes one argument, the butler, and uses
691 stars and visits previously extracted from dataRefs by
692 fgcmBuildStars.
693 This Runner does not perform any dataRef parallelization, but the FGCM
694 code called by the Task uses python multiprocessing (see the "ncores"
695 config option).
696 """
698 @staticmethod
699 def getTargetList(parsedCmd):
700 """
701 Return a list with one element, the butler.
702 """
703 return [parsedCmd.butler]
705 def __call__(self, butler):
706 """
707 Parameters
708 ----------
709 butler: `lsst.daf.persistence.Butler`
711 Returns
712 -------
713 exitStatus: `list` with `pipeBase.Struct`
714 exitStatus (0: success; 1: failure)
715 """
717 task = self.TaskClass(config=self.config, log=self.log)
719 exitStatus = 0
720 if self.doRaise:
721 task.runDataRef(butler)
722 else:
723 try:
724 task.runDataRef(butler)
725 except Exception as e:
726 exitStatus = 1
727 task.log.fatal("Failed: %s" % e)
728 if not isinstance(e, pipeBase.TaskError):
729 traceback.print_exc(file=sys.stderr)
731 task.writeMetadata(butler)
733 # The task does not return any results:
734 return [pipeBase.Struct(exitStatus=exitStatus)]
736 def run(self, parsedCmd):
737 """
738 Run the task, with no multiprocessing
740 Parameters
741 ----------
742 parsedCmd: ArgumentParser parsed command line
743 """
745 resultList = []
747 if self.precall(parsedCmd):
748 targetList = self.getTargetList(parsedCmd)
749 # make sure that we only get 1
750 resultList = self(targetList[0])
752 return resultList
755class FgcmFitCycleTask(pipeBase.CmdLineTask):
756 """
757 Run Single fit cycle for FGCM global calibration
758 """
760 ConfigClass = FgcmFitCycleConfig
761 RunnerClass = FgcmFitCycleRunner
762 _DefaultName = "fgcmFitCycle"
764 def __init__(self, butler=None, **kwargs):
765 """
766 Instantiate an fgcmFitCycle.
768 Parameters
769 ----------
770 butler : `lsst.daf.persistence.Butler`
771 """
773 pipeBase.CmdLineTask.__init__(self, **kwargs)
775 # no saving of metadata for now
776 def _getMetadataName(self):
777 return None
779 @pipeBase.timeMethod
780 def runDataRef(self, butler):
781 """
782 Run a single fit cycle for FGCM
784 Parameters
785 ----------
786 butler: `lsst.daf.persistence.Butler`
787 """
789 self._fgcmFitCycle(butler)
791 def writeConfig(self, butler, clobber=False, doBackup=True):
792 """Write the configuration used for processing the data, or check that an existing
793 one is equal to the new one if present. This is an override of the regular
794 version from pipe_base that knows about fgcmcycle.
796 Parameters
797 ----------
798 butler : `lsst.daf.persistence.Butler`
799 Data butler used to write the config. The config is written to dataset type
800 `CmdLineTask._getConfigName`.
801 clobber : `bool`, optional
802 A boolean flag that controls what happens if a config already has been saved:
803 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
804 - `False`: raise `TaskError` if this config does not match the existing config.
805 doBackup : `bool`, optional
806 Set to `True` to backup the config files if clobbering.
807 """
808 configName = self._getConfigName()
809 if configName is None:
810 return
811 if clobber:
812 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
813 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
814 # this may be subject to a race condition; see #2789
815 try:
816 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
817 except Exception as exc:
818 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
819 (configName, exc))
821 def logConfigMismatch(msg):
822 self.log.fatal("Comparing configuration: %s", msg)
824 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
825 raise pipeBase.TaskError(
826 ("Config does not match existing task config %r on disk; tasks configurations " +
827 "must be consistent within the same output repo (override with --clobber-config)") %
828 (configName,))
829 else:
830 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
832 def _fgcmFitCycle(self, butler):
833 """
834 Run the fit cycle
836 Parameters
837 ----------
838 butler: `lsst.daf.persistence.Butler`
839 """
841 self._checkDatasetsExist(butler)
843 # Set defaults on whether to output standards and zeropoints
844 self.maxIter = self.config.maxIterBeforeFinalCycle
845 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
846 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
847 self.resetFitParameters = True
849 if self.config.isFinalCycle:
850 # This is the final fit cycle, so we do not want to reset fit
851 # parameters, we want to run a final "clean-up" with 0 fit iterations,
852 # and we always want to output standards and zeropoints
853 self.maxIter = 0
854 self.outputStandards = True
855 self.outputZeropoints = True
856 self.resetFitParameters = False
858 camera = butler.get('camera')
859 configDict = makeConfigDict(self.config, self.log, camera,
860 self.maxIter, self.resetFitParameters,
861 self.outputZeropoints)
863 lutCat = butler.get('fgcmLookUpTable')
864 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
865 del lutCat
867 # next we need the exposure/visit information
869 # fgcmExpInfo = self._loadVisitCatalog(butler)
870 visitCat = butler.get('fgcmVisitCatalog')
871 fgcmExpInfo = translateVisitCatalog(visitCat)
872 del visitCat
874 # Use the first orientation.
875 # TODO: DM-21215 will generalize to arbitrary camera orientations
876 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
878 noFitsDict = {'lutIndex': lutIndexVals,
879 'lutStd': lutStd,
880 'expInfo': fgcmExpInfo,
881 'ccdOffsets': ccdOffsets}
883 # set up the fitter object
884 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
885 noFitsDict=noFitsDict, noOutput=True)
887 # create the parameter object
888 if (fgcmFitCycle.initialCycle):
889 # cycle = 0, initial cycle
890 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
891 fgcmLut,
892 fgcmExpInfo)
893 else:
894 inParInfo, inParams, inSuperStar = self._loadParameters(butler)
895 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
896 fgcmExpInfo,
897 inParInfo,
898 inParams,
899 inSuperStar)
901 lastCycle = configDict['cycleNumber'] - 1
903 # set up the stars...
904 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
906 starObs = butler.get('fgcmStarObservations')
907 starIds = butler.get('fgcmStarIds')
908 starIndices = butler.get('fgcmStarIndices')
910 # grab the flagged stars if available
911 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
912 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
913 flagId = flaggedStars['objId'][:]
914 flagFlag = flaggedStars['objFlag'][:]
915 else:
916 flaggedStars = None
917 flagId = None
918 flagFlag = None
920 if self.config.doReferenceCalibration:
921 refStars = butler.get('fgcmReferenceStars')
923 refMag, refMagErr = extractReferenceMags(refStars,
924 self.config.bands,
925 self.config.filterMap)
926 refId = refStars['fgcm_id'][:]
927 else:
928 refStars = None
929 refId = None
930 refMag = None
931 refMagErr = None
933 # match star observations to visits
934 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
935 # actually be transferred into fgcm using the indexing below.
936 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
938 # The fgcmStars.loadStars method will copy all the star information into
939 # special shared memory objects that will not blow up the memory usage when
940 # used with python multiprocessing. Once all the numbers are copied,
941 # it is necessary to release all references to the objects that previously
942 # stored the data to ensure that the garbage collector can clear the memory,
943 # and ensure that this memory is not copied when multiprocessing kicks in.
945 # We determine the conversion from the native units (typically radians) to
946 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
947 # numpy arrays rather than Angles, which would we approximately 600x slower.
948 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
950 fgcmStars.loadStars(fgcmPars,
951 starObs['visit'][starIndices['obsIndex']],
952 starObs['ccd'][starIndices['obsIndex']],
953 starObs['ra'][starIndices['obsIndex']] * conv,
954 starObs['dec'][starIndices['obsIndex']] * conv,
955 starObs['instMag'][starIndices['obsIndex']],
956 starObs['instMagErr'][starIndices['obsIndex']],
957 fgcmExpInfo['FILTERNAME'][visitIndex],
958 starIds['fgcm_id'][:],
959 starIds['ra'][:],
960 starIds['dec'][:],
961 starIds['obsArrIndex'][:],
962 starIds['nObs'][:],
963 obsX=starObs['x'][starIndices['obsIndex']],
964 obsY=starObs['y'][starIndices['obsIndex']],
965 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
966 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
967 refID=refId,
968 refMag=refMag,
969 refMagErr=refMagErr,
970 flagID=flagId,
971 flagFlag=flagFlag,
972 computeNobs=True)
974 # Release all references to temporary objects holding star data (see above)
975 del starObs
976 del starIds
977 del starIndices
978 del flagId
979 del flagFlag
980 del flaggedStars
981 del refStars
982 del refId
983 del refMag
984 del refMagErr
986 # and set the bits in the cycle object
987 fgcmFitCycle.setLUT(fgcmLut)
988 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
989 fgcmFitCycle.setPars(fgcmPars)
991 # finish the setup
992 fgcmFitCycle.finishSetup()
994 # and run
995 fgcmFitCycle.run()
997 ##################
998 # Persistance
999 ##################
1001 self._persistFgcmDatasets(butler, fgcmFitCycle)
1003 # Output the config for the next cycle
1004 # We need to make a copy since the input one has been frozen
1006 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1007 i, b in enumerate(self.config.bands)}
1008 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1009 i, band in enumerate(self.config.bands)}
1011 outConfig = copy.copy(self.config)
1012 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
1013 precomputeSuperStarInitialCycle=False,
1014 freezeStdAtmosphere=False,
1015 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1016 expGrayHighCutDict=updatedHighCutDict)
1017 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1018 outConfig.cycleNumber)
1019 outConfig.save(configFileName)
1021 if self.config.isFinalCycle == 1:
1022 # We are done, ready to output products
1023 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1024 else:
1025 self.log.info("Saved config for next cycle to %s" % (configFileName))
1026 self.log.info("Be sure to look at:")
1027 self.log.info(" config.expGrayPhotometricCut")
1028 self.log.info(" config.expGrayHighCut")
1029 self.log.info("If you are satisfied with the fit, please set:")
1030 self.log.info(" config.isFinalCycle = True")
1032 def _checkDatasetsExist(self, butler):
1033 """
1034 Check if necessary datasets exist to run fgcmFitCycle
1036 Parameters
1037 ----------
1038 butler: `lsst.daf.persistence.Butler`
1040 Raises
1041 ------
1042 RuntimeError
1043 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1044 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1045 If cycleNumber > 0, then also checks for fgcmFitParameters,
1046 fgcmFlaggedStars.
1047 """
1049 if not butler.datasetExists('fgcmVisitCatalog'):
1050 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1051 if not butler.datasetExists('fgcmStarObservations'):
1052 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1053 if not butler.datasetExists('fgcmStarIds'):
1054 raise RuntimeError("Could not find fgcmStarIds in repo!")
1055 if not butler.datasetExists('fgcmStarIndices'):
1056 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1057 if not butler.datasetExists('fgcmLookUpTable'):
1058 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1060 # Need additional datasets if we are not the initial cycle
1061 if (self.config.cycleNumber > 0):
1062 if not butler.datasetExists('fgcmFitParameters',
1063 fgcmcycle=self.config.cycleNumber-1):
1064 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1065 (self.config.cycleNumber-1))
1066 if not butler.datasetExists('fgcmFlaggedStars',
1067 fgcmcycle=self.config.cycleNumber-1):
1068 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1069 (self.config.cycleNumber-1))
1071 # And additional dataset if we want reference calibration
1072 if self.config.doReferenceCalibration:
1073 if not butler.datasetExists('fgcmReferenceStars'):
1074 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1075 "doReferenceCalibration is True.")
1077 def _loadParameters(self, butler):
1078 """
1079 Load FGCM parameters from a previous fit cycle
1081 Parameters
1082 ----------
1083 butler: `lsst.daf.persistence.Butler`
1085 Returns
1086 -------
1087 inParInfo: `numpy.ndarray`
1088 Numpy array parameter information formatted for input to fgcm
1089 inParameters: `numpy.ndarray`
1090 Numpy array parameter values formatted for input to fgcm
1091 inSuperStar: `numpy.array`
1092 Superstar flat formatted for input to fgcm
1093 """
1095 # note that we already checked that this is available
1096 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1098 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1099 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1101 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1102 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1103 (parLutFilterNames.size, )),
1104 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1105 ('LNTAUUNIT', 'f8'),
1106 ('LNTAUSLOPEUNIT', 'f8'),
1107 ('ALPHAUNIT', 'f8'),
1108 ('LNPWVUNIT', 'f8'),
1109 ('LNPWVSLOPEUNIT', 'f8'),
1110 ('LNPWVQUADRATICUNIT', 'f8'),
1111 ('LNPWVGLOBALUNIT', 'f8'),
1112 ('O3UNIT', 'f8'),
1113 ('QESYSUNIT', 'f8'),
1114 ('FILTEROFFSETUNIT', 'f8'),
1115 ('HASEXTERNALPWV', 'i2'),
1116 ('HASEXTERNALTAU', 'i2')])
1117 inParInfo['NCCD'] = parCat['nCcd']
1118 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1119 inParInfo['FITBANDS'][:] = parFitBands
1120 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1121 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1123 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1124 ('PARO3', 'f8', (parCat['parO3'].size, )),
1125 ('PARLNTAUINTERCEPT', 'f8',
1126 (parCat['parLnTauIntercept'].size, )),
1127 ('PARLNTAUSLOPE', 'f8',
1128 (parCat['parLnTauSlope'].size, )),
1129 ('PARLNPWVINTERCEPT', 'f8',
1130 (parCat['parLnPwvIntercept'].size, )),
1131 ('PARLNPWVSLOPE', 'f8',
1132 (parCat['parLnPwvSlope'].size, )),
1133 ('PARLNPWVQUADRATIC', 'f8',
1134 (parCat['parLnPwvQuadratic'].size, )),
1135 ('PARQESYSINTERCEPT', 'f8',
1136 (parCat['parQeSysIntercept'].size, )),
1137 ('COMPQESYSSLOPE', 'f8',
1138 (parCat['compQeSysSlope'].size, )),
1139 ('PARFILTEROFFSET', 'f8',
1140 (parCat['parFilterOffset'].size, )),
1141 ('PARFILTEROFFSETFITFLAG', 'i2',
1142 (parCat['parFilterOffsetFitFlag'].size, )),
1143 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1144 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1145 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1146 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1147 ('COMPABSTHROUGHPUT', 'f8',
1148 (parCat['compAbsThroughput'].size, )),
1149 ('COMPREFOFFSET', 'f8',
1150 (parCat['compRefOffset'].size, )),
1151 ('COMPREFSIGMA', 'f8',
1152 (parCat['compRefSigma'].size, )),
1153 ('COMPMIRRORCHROMATICITY', 'f8',
1154 (parCat['compMirrorChromaticity'].size, )),
1155 ('MIRRORCHROMATICITYPIVOT', 'f8',
1156 (parCat['mirrorChromaticityPivot'].size, )),
1157 ('COMPMEDIANSEDSLOPE', 'f8',
1158 (parCat['compMedianSedSlope'].size, )),
1159 ('COMPAPERCORRPIVOT', 'f8',
1160 (parCat['compAperCorrPivot'].size, )),
1161 ('COMPAPERCORRSLOPE', 'f8',
1162 (parCat['compAperCorrSlope'].size, )),
1163 ('COMPAPERCORRSLOPEERR', 'f8',
1164 (parCat['compAperCorrSlopeErr'].size, )),
1165 ('COMPAPERCORRRANGE', 'f8',
1166 (parCat['compAperCorrRange'].size, )),
1167 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1168 (parCat['compModelErrExptimePivot'].size, )),
1169 ('COMPMODELERRFWHMPIVOT', 'f8',
1170 (parCat['compModelErrFwhmPivot'].size, )),
1171 ('COMPMODELERRSKYPIVOT', 'f8',
1172 (parCat['compModelErrSkyPivot'].size, )),
1173 ('COMPMODELERRPARS', 'f8',
1174 (parCat['compModelErrPars'].size, )),
1175 ('COMPEXPGRAY', 'f8',
1176 (parCat['compExpGray'].size, )),
1177 ('COMPVARGRAY', 'f8',
1178 (parCat['compVarGray'].size, )),
1179 ('COMPEXPDELTAMAGBKG', 'f8',
1180 (parCat['compExpDeltaMagBkg'].size, )),
1181 ('COMPNGOODSTARPEREXP', 'i4',
1182 (parCat['compNGoodStarPerExp'].size, )),
1183 ('COMPSIGFGCM', 'f8',
1184 (parCat['compSigFgcm'].size, )),
1185 ('COMPSIGMACAL', 'f8',
1186 (parCat['compSigmaCal'].size, )),
1187 ('COMPRETRIEVEDLNPWV', 'f8',
1188 (parCat['compRetrievedLnPwv'].size, )),
1189 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1190 (parCat['compRetrievedLnPwvRaw'].size, )),
1191 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1192 (parCat['compRetrievedLnPwvFlag'].size, )),
1193 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1194 (parCat['compRetrievedTauNight'].size, ))])
1196 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1197 inParams['PARO3'][:] = parCat['parO3'][0, :]
1198 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1199 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1200 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1201 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1202 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1203 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1204 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1205 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1206 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1207 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1208 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1209 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1210 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1211 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1212 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1213 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1214 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1215 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1216 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1217 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1218 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1219 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1220 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1221 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1222 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1223 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1224 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1225 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1226 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1227 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1228 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1229 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1230 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1231 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1232 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1233 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1235 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1236 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1238 return (inParInfo, inParams, inSuperStar)
1240 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1241 """
1242 Persist FGCM datasets through the butler.
1244 Parameters
1245 ----------
1246 butler: `lsst.daf.persistence.Butler`
1247 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1248 Fgcm Fit cycle object
1249 """
1251 # Save the parameters
1252 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1254 parSchema = afwTable.Schema()
1256 comma = ','
1257 lutFilterNameString = comma.join([n.decode('utf-8')
1258 for n in parInfo['LUTFILTERNAMES'][0]])
1259 fitBandString = comma.join([n.decode('utf-8')
1260 for n in parInfo['FITBANDS'][0]])
1262 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1263 lutFilterNameString, fitBandString)
1264 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1265 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1266 lutFilterNameString, fitBandString)
1268 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1270 # Save the indices of the flagged stars
1271 # (stars that have been (a) reserved from the fit for testing and
1272 # (b) bad stars that have failed quality checks.)
1273 flagStarSchema = self._makeFlagStarSchema()
1274 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1275 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1277 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1279 # Save the zeropoint information and atmospheres only if desired
1280 if self.outputZeropoints:
1281 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1282 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1284 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1285 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1287 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1289 # Save atmosphere values
1290 # These are generated by the same code that generates zeropoints
1291 atmSchema = makeAtmSchema()
1292 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1294 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1296 # Save the standard stars (if configured)
1297 if self.outputStandards:
1298 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1299 stdSchema = makeStdSchema(len(goodBands))
1300 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1302 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1304 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1305 lutFilterNameString, fitBandString):
1306 """
1307 Make the parameter persistence schema
1309 Parameters
1310 ----------
1311 parInfo: `numpy.ndarray`
1312 Parameter information returned by fgcm
1313 pars: `numpy.ndarray`
1314 Parameter values returned by fgcm
1315 parSuperStarFlat: `numpy.array`
1316 Superstar flat values returned by fgcm
1317 lutFilterNameString: `str`
1318 Combined string of all the lutFilterNames
1319 fitBandString: `str`
1320 Combined string of all the fitBands
1322 Returns
1323 -------
1324 parSchema: `afwTable.schema`
1325 """
1327 parSchema = afwTable.Schema()
1329 # parameter info section
1330 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1331 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1332 size=len(lutFilterNameString))
1333 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1334 size=len(fitBandString))
1335 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1336 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1337 doc='Step units for ln(AOD) slope')
1338 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1339 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1340 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1341 doc='Step units for ln(pwv) slope')
1342 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1343 doc='Step units for ln(pwv) quadratic term')
1344 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1345 doc='Step units for global ln(pwv) parameters')
1346 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1347 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1348 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1349 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1350 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1352 # parameter section
1353 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1354 size=pars['PARALPHA'].size)
1355 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1356 size=pars['PARO3'].size)
1357 parSchema.addField('parLnTauIntercept', type='ArrayD',
1358 doc='ln(Tau) intercept parameter vector',
1359 size=pars['PARLNTAUINTERCEPT'].size)
1360 parSchema.addField('parLnTauSlope', type='ArrayD',
1361 doc='ln(Tau) slope parameter vector',
1362 size=pars['PARLNTAUSLOPE'].size)
1363 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1364 size=pars['PARLNPWVINTERCEPT'].size)
1365 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1366 size=pars['PARLNPWVSLOPE'].size)
1367 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1368 size=pars['PARLNPWVQUADRATIC'].size)
1369 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1370 size=pars['PARQESYSINTERCEPT'].size)
1371 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1372 size=pars[0]['COMPQESYSSLOPE'].size)
1373 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1374 size=pars['PARFILTEROFFSET'].size)
1375 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1376 size=pars['PARFILTEROFFSETFITFLAG'].size)
1377 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1378 doc='Global scale for retrieved ln(pwv)')
1379 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1380 doc='Global offset for retrieved ln(pwv)')
1381 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1382 doc='Nightly offset for retrieved ln(pwv)',
1383 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1384 parSchema.addField('compAbsThroughput', type='ArrayD',
1385 doc='Absolute throughput (relative to transmission curves)',
1386 size=pars['COMPABSTHROUGHPUT'].size)
1387 parSchema.addField('compRefOffset', type='ArrayD',
1388 doc='Offset between reference stars and calibrated stars',
1389 size=pars['COMPREFOFFSET'].size)
1390 parSchema.addField('compRefSigma', type='ArrayD',
1391 doc='Width of reference star/calibrated star distribution',
1392 size=pars['COMPREFSIGMA'].size)
1393 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1394 doc='Computed mirror chromaticity terms',
1395 size=pars['COMPMIRRORCHROMATICITY'].size)
1396 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1397 doc='Mirror chromaticity pivot mjd',
1398 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1399 parSchema.addField('compMedianSedSlope', type='ArrayD',
1400 doc='Computed median SED slope (per band)',
1401 size=pars['COMPMEDIANSEDSLOPE'].size)
1402 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1403 size=pars['COMPAPERCORRPIVOT'].size)
1404 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1405 size=pars['COMPAPERCORRSLOPE'].size)
1406 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1407 size=pars['COMPAPERCORRSLOPEERR'].size)
1408 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1409 size=pars['COMPAPERCORRRANGE'].size)
1410 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1411 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1412 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1413 size=pars['COMPMODELERRFWHMPIVOT'].size)
1414 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1415 size=pars['COMPMODELERRSKYPIVOT'].size)
1416 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1417 size=pars['COMPMODELERRPARS'].size)
1418 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1419 size=pars['COMPEXPGRAY'].size)
1420 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1421 size=pars['COMPVARGRAY'].size)
1422 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1423 doc='Computed exposure offset due to background',
1424 size=pars['COMPEXPDELTAMAGBKG'].size)
1425 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1426 doc='Computed number of good stars per exposure',
1427 size=pars['COMPNGOODSTARPEREXP'].size)
1428 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1429 size=pars['COMPSIGFGCM'].size)
1430 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1431 size=pars['COMPSIGMACAL'].size)
1432 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1433 size=pars['COMPRETRIEVEDLNPWV'].size)
1434 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1435 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1436 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1437 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1438 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1439 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1440 # superstarflat section
1441 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1442 size=4)
1443 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1444 size=parSuperStarFlat.size)
1446 return parSchema
1448 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1449 lutFilterNameString, fitBandString):
1450 """
1451 Make the FGCM parameter catalog for persistence
1453 Parameters
1454 ----------
1455 parSchema: `lsst.afw.table.Schema`
1456 Parameter catalog schema
1457 pars: `numpy.ndarray`
1458 FGCM parameters to put into parCat
1459 parSuperStarFlat: `numpy.array`
1460 FGCM superstar flat array to put into parCat
1461 lutFilterNameString: `str`
1462 Combined string of all the lutFilterNames
1463 fitBandString: `str`
1464 Combined string of all the fitBands
1466 Returns
1467 -------
1468 parCat: `afwTable.BasicCatalog`
1469 Atmosphere and instrumental model parameter catalog for persistence
1470 """
1472 parCat = afwTable.BaseCatalog(parSchema)
1473 parCat.reserve(1)
1475 # The parameter catalog just has one row, with many columns for all the
1476 # atmosphere and instrument fit parameters
1477 rec = parCat.addNew()
1479 # info section
1480 rec['nCcd'] = parInfo['NCCD']
1481 rec['lutFilterNames'] = lutFilterNameString
1482 rec['fitBands'] = fitBandString
1483 # note these are not currently supported here.
1484 rec['hasExternalPwv'] = 0
1485 rec['hasExternalTau'] = 0
1487 # parameter section
1489 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1491 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1492 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1493 'parQeSysIntercept', 'compQeSysSlope',
1494 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1495 'parFilterOffset', 'parFilterOffsetFitFlag',
1496 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1497 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1498 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1499 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1500 'compModelErrSkyPivot', 'compModelErrPars',
1501 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1502 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1503 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1504 'compRetrievedTauNight']
1506 for scalarName in scalarNames:
1507 rec[scalarName] = pars[scalarName.upper()]
1509 for arrName in arrNames:
1510 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1512 # superstar section
1513 rec['superstarSize'][:] = parSuperStarFlat.shape
1514 rec['superstar'][:] = parSuperStarFlat.flatten()
1516 return parCat
1518 def _makeFlagStarSchema(self):
1519 """
1520 Make the flagged-stars schema
1522 Returns
1523 -------
1524 flagStarSchema: `lsst.afw.table.Schema`
1525 """
1527 flagStarSchema = afwTable.Schema()
1529 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1530 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1532 return flagStarSchema
1534 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1535 """
1536 Make the flagged star catalog for persistence
1538 Parameters
1539 ----------
1540 flagStarSchema: `lsst.afw.table.Schema`
1541 Flagged star schema
1542 flagStarStruct: `numpy.ndarray`
1543 Flagged star structure from fgcm
1545 Returns
1546 -------
1547 flagStarCat: `lsst.afw.table.BaseCatalog`
1548 Flagged star catalog for persistence
1549 """
1551 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1552 flagStarCat.resize(flagStarStruct.size)
1554 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1555 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1557 return flagStarCat