Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44import lsst.afw.table as afwTable
46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47from .utilities import extractReferenceMags
48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50from .sedterms import SedboundarytermDict, SedtermDict
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57class FgcmFitCycleConfig(pexConfig.Config):
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc="Bands to run calibration",
62 dtype=str,
63 default=[],
64 )
65 fitFlag = pexConfig.ListField(
66 doc=("Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
70 dtype=int,
71 default=(0,),
72 optional=True,
73 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
74 "It will be removed after v20. Use fitBands instead."),
75 )
76 fitBands = pexConfig.ListField(
77 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
78 "the atmosphere constrained from the 'fitBands' on the same night. "
79 "Must be a subset of `config.bands`"),
80 dtype=str,
81 default=[],
82 )
83 requiredFlag = pexConfig.ListField(
84 doc=("Flag for which bands are required for a star to be considered a calibration "
85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86 "be same length as config.bands, and matched band-by-band."),
87 dtype=int,
88 default=(0,),
89 optional=True,
90 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
91 "It will be removed after v20. Use requiredBands instead."),
92 )
93 requiredBands = pexConfig.ListField(
94 doc=("Bands that are required for a star to be considered a calibration star. "
95 "Must be a subset of `config.bands`"),
96 dtype=str,
97 default=[],
98 )
99 filterMap = pexConfig.DictField(
100 doc="Mapping from 'filterName' to band.",
101 keytype=str,
102 itemtype=str,
103 default={},
104 )
105 doReferenceCalibration = pexConfig.Field(
106 doc="Use reference catalog as additional constraint on calibration",
107 dtype=bool,
108 default=True,
109 )
110 refStarSnMin = pexConfig.Field(
111 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
112 dtype=float,
113 default=50.0,
114 )
115 refStarOutlierNSig = pexConfig.Field(
116 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
118 dtype=float,
119 default=4.0,
120 )
121 applyRefStarColorCuts = pexConfig.Field(
122 doc="Apply color cuts to reference stars?",
123 dtype=bool,
124 default=True,
125 )
126 nCore = pexConfig.Field(
127 doc="Number of cores to use",
128 dtype=int,
129 default=4,
130 )
131 nStarPerRun = pexConfig.Field(
132 doc="Number of stars to run in each chunk",
133 dtype=int,
134 default=200000,
135 )
136 nExpPerRun = pexConfig.Field(
137 doc="Number of exposures to run in each chunk",
138 dtype=int,
139 default=1000,
140 )
141 reserveFraction = pexConfig.Field(
142 doc="Fraction of stars to reserve for testing",
143 dtype=float,
144 default=0.1,
145 )
146 freezeStdAtmosphere = pexConfig.Field(
147 doc="Freeze atmosphere parameters to standard (for testing)",
148 dtype=bool,
149 default=False,
150 )
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc="Precompute superstar flat for initial cycle",
153 dtype=bool,
154 default=False,
155 )
156 superStarSubCcd = pexConfig.Field(
157 doc="Compute superstar flat on sub-ccd scale",
158 dtype=bool,
159 default=True,
160 optional=True,
161 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
162 "It will be removed after v20. Use superStarSubCcdDict instead."),
163 )
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166 "Must have one entry per band."),
167 keytype=str,
168 itemtype=bool,
169 default={},
170 )
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173 "Global default is first-order polynomials, and should be overridden "
174 "on a camera-by-camera basis depending on the ISR."),
175 dtype=int,
176 default=1,
177 )
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
180 "suppress high-order cross terms?"),
181 dtype=bool,
182 default=False,
183 )
184 superStarSigmaClip = pexConfig.Field(
185 doc="Number of sigma to clip outliers when selecting for superstar flats",
186 dtype=float,
187 default=5.0,
188 )
189 ccdGraySubCcd = pexConfig.Field(
190 doc="Compute CCD gray terms on sub-ccd scale",
191 dtype=bool,
192 default=False,
193 optional=True,
194 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
195 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
196 )
197 ccdGraySubCcdDict = pexConfig.DictField(
198 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
199 "('ccd gray') on a sub-ccd scale."),
200 keytype=str,
201 itemtype=bool,
202 default={},
203 )
204 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
206 dtype=int,
207 default=1,
208 )
209 ccdGraySubCcdTriangular = pexConfig.Field(
210 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
211 "suppress high-order cross terms?"),
212 dtype=bool,
213 default=True,
214 )
215 cycleNumber = pexConfig.Field(
216 doc=("FGCM fit cycle number. This is automatically incremented after each run "
217 "and stage of outlier rejection. See cookbook for details."),
218 dtype=int,
219 default=None,
220 )
221 isFinalCycle = pexConfig.Field(
222 doc=("Is this the final cycle of the fitting? Will automatically compute final "
223 "selection of stars and photometric exposures, and will output zeropoints "
224 "and standard stars for use in fgcmOutputProducts"),
225 dtype=bool,
226 default=False,
227 )
228 maxIterBeforeFinalCycle = pexConfig.Field(
229 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
230 "will always be 0 in the final cycle for cleanup and final selection."),
231 dtype=int,
232 default=50,
233 )
234 deltaMagBkgOffsetPercentile = pexConfig.Field(
235 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
236 "offset from local background subtraction."),
237 dtype=float,
238 default=0.25,
239 )
240 deltaMagBkgPerCcd = pexConfig.Field(
241 doc=("Compute net offset from local background subtraction per-ccd? "
242 "Otherwise, use computation per visit."),
243 dtype=bool,
244 default=False,
245 )
246 utBoundary = pexConfig.Field(
247 doc="Boundary (in UTC) from day-to-day",
248 dtype=float,
249 default=None,
250 )
251 washMjds = pexConfig.ListField(
252 doc="Mirror wash MJDs",
253 dtype=float,
254 default=(0.0,),
255 )
256 epochMjds = pexConfig.ListField(
257 doc="Epoch boundaries in MJD",
258 dtype=float,
259 default=(0.0,),
260 )
261 minObsPerBand = pexConfig.Field(
262 doc="Minimum good observations per band",
263 dtype=int,
264 default=2,
265 )
266 # TODO: When DM-16511 is done, it will be possible to get the
267 # telescope latitude directly from the camera.
268 latitude = pexConfig.Field(
269 doc="Observatory latitude",
270 dtype=float,
271 default=None,
272 )
273 brightObsGrayMax = pexConfig.Field(
274 doc="Maximum gray extinction to be considered bright observation",
275 dtype=float,
276 default=0.15,
277 )
278 minStarPerCcd = pexConfig.Field(
279 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
280 "CCDs with fewer stars will have their calibration estimated from other "
281 "CCDs in the same visit, with zeropoint error increased accordingly."),
282 dtype=int,
283 default=5,
284 )
285 minCcdPerExp = pexConfig.Field(
286 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
287 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
288 dtype=int,
289 default=5,
290 )
291 maxCcdGrayErr = pexConfig.Field(
292 doc="Maximum error on CCD gray offset to be considered photometric",
293 dtype=float,
294 default=0.05,
295 )
296 minStarPerExp = pexConfig.Field(
297 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
298 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
299 dtype=int,
300 default=600,
301 )
302 minExpPerNight = pexConfig.Field(
303 doc="Minimum number of good exposures/visits to consider a partly photometric night",
304 dtype=int,
305 default=10,
306 )
307 expGrayInitialCut = pexConfig.Field(
308 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
309 "observations."),
310 dtype=float,
311 default=-0.25,
312 )
313 expGrayPhotometricCut = pexConfig.ListField(
314 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
315 "Must be same length as config.bands, and matched band-by-band."),
316 dtype=float,
317 default=(0.0,),
318 optional=True,
319 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
320 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
321 )
322 expGrayPhotometricCutDict = pexConfig.DictField(
323 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
324 "('gray term') for a visit to be considered photometric. Must have one "
325 "entry per band. Broad-band filters should be -0.05."),
326 keytype=str,
327 itemtype=float,
328 default={},
329 )
330 expGrayHighCut = pexConfig.ListField(
331 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
332 "Must be same length as config.bands, and matched band-by-band."),
333 dtype=float,
334 default=(0.0,),
335 optional=True,
336 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
337 "It will be removed after v20. Use expGrayHighCutDict instead."),
338 )
339 expGrayHighCutDict = pexConfig.DictField(
340 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
341 "('gray term') for a visit to be considered photometric. Must have one "
342 "entry per band. Broad-band filters should be 0.2."),
343 keytype=str,
344 itemtype=float,
345 default={},
346 )
347 expGrayRecoverCut = pexConfig.Field(
348 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
349 "Visits with more gray extinction will only get CCD zeropoints if there are "
350 "sufficient star observations (minStarPerCcd) on that CCD."),
351 dtype=float,
352 default=-1.0,
353 )
354 expVarGrayPhotometricCut = pexConfig.Field(
355 doc="Maximum exposure variance to be considered possibly photometric",
356 dtype=float,
357 default=0.0005,
358 optional=True,
359 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
360 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
361 )
362 expVarGrayPhotometricCutDict = pexConfig.DictField(
363 doc=("Per-band specification on maximum exposure variance to be considered possibly "
364 "photometric. Must have one entry per band. Broad-band filters should be "
365 "0.0005."),
366 keytype=str,
367 itemtype=float,
368 default={},
369 )
370 expGrayErrRecoverCut = pexConfig.Field(
371 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
372 "Visits with more gray variance will only get CCD zeropoints if there are "
373 "sufficient star observations (minStarPerCcd) on that CCD."),
374 dtype=float,
375 default=0.05,
376 )
377 aperCorrFitNBins = pexConfig.Field(
378 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
379 "no fit will be performed, and the config.aperCorrInputSlopes will be "
380 "used if available."),
381 dtype=int,
382 default=10,
383 )
384 aperCorrInputSlopes = pexConfig.ListField(
385 doc=("Aperture correction input slope parameters. These are used on the first "
386 "fit iteration, and aperture correction parameters will be updated from "
387 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
388 "when there is insufficient data to fit the parameters (e.g. tract mode). "
389 "If set, must be same length as config.bands, and matched band-by-band."),
390 dtype=float,
391 default=[],
392 optional=True,
393 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
394 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
395 )
396 aperCorrInputSlopeDict = pexConfig.DictField(
397 doc=("Per-band specification of aperture correction input slope parameters. These "
398 "are used on the first fit iteration, and aperture correction parameters will "
399 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
400 "to set this when there is insufficient data to fit the parameters (e.g. "
401 "tract mode)."),
402 keytype=str,
403 itemtype=float,
404 default={},
405 )
406 sedFudgeFactors = pexConfig.ListField(
407 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
408 "config.bands, and matched band-by-band."),
409 dtype=float,
410 default=(0,),
411 optional=True,
412 deprecated=("This field has been deprecated and will be removed after v20. "
413 "Please use sedSlopeTermMap and sedSlopeMap."),
414 )
415 sedboundaryterms = pexConfig.ConfigField(
416 doc="Mapping from bands to SED boundary term names used is sedterms.",
417 dtype=SedboundarytermDict,
418 )
419 sedterms = pexConfig.ConfigField(
420 doc="Mapping from terms to bands for fgcm linear SED approximations.",
421 dtype=SedtermDict,
422 )
423 sigFgcmMaxErr = pexConfig.Field(
424 doc="Maximum mag error for fitting sigma_FGCM",
425 dtype=float,
426 default=0.01,
427 )
428 sigFgcmMaxEGray = pexConfig.ListField(
429 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
430 "May be 1 element (same for all bands) or the same length as config.bands."),
431 dtype=float,
432 default=(0.05,),
433 optional=True,
434 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
435 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
436 )
437 sigFgcmMaxEGrayDict = pexConfig.DictField(
438 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
439 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
440 "should be 0.05."),
441 keytype=str,
442 itemtype=float,
443 default={},
444 )
445 ccdGrayMaxStarErr = pexConfig.Field(
446 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
447 "computation"),
448 dtype=float,
449 default=0.10,
450 )
451 approxThroughput = pexConfig.ListField(
452 doc=("Approximate overall throughput at start of calibration observations. "
453 "May be 1 element (same for all bands) or the same length as config.bands."),
454 dtype=float,
455 default=(1.0, ),
456 optional=True,
457 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
458 "It will be removed after v20. Use approxThroughputDict instead."),
459 )
460 approxThroughputDict = pexConfig.DictField(
461 doc=("Per-band specification of the approximate overall throughput at the start of "
462 "calibration observations. Must have one entry per band. Typically should "
463 "be 1.0."),
464 keytype=str,
465 itemtype=float,
466 default={},
467 )
468 sigmaCalRange = pexConfig.ListField(
469 doc="Allowed range for systematic error floor estimation",
470 dtype=float,
471 default=(0.001, 0.003),
472 )
473 sigmaCalFitPercentile = pexConfig.ListField(
474 doc="Magnitude percentile range to fit systematic error floor",
475 dtype=float,
476 default=(0.05, 0.15),
477 )
478 sigmaCalPlotPercentile = pexConfig.ListField(
479 doc="Magnitude percentile range to plot systematic error floor",
480 dtype=float,
481 default=(0.05, 0.95),
482 )
483 sigma0Phot = pexConfig.Field(
484 doc="Systematic error floor for all zeropoints",
485 dtype=float,
486 default=0.003,
487 )
488 mapLongitudeRef = pexConfig.Field(
489 doc="Reference longitude for plotting maps",
490 dtype=float,
491 default=0.0,
492 )
493 mapNSide = pexConfig.Field(
494 doc="Healpix nside for plotting maps",
495 dtype=int,
496 default=256,
497 )
498 outfileBase = pexConfig.Field(
499 doc="Filename start for plot output files",
500 dtype=str,
501 default=None,
502 )
503 starColorCuts = pexConfig.ListField(
504 doc="Encoded star-color cuts (to be cleaned up)",
505 dtype=str,
506 default=("NO_DATA",),
507 )
508 colorSplitIndices = pexConfig.ListField(
509 doc="Band indices to use to split stars by color",
510 dtype=int,
511 default=None,
512 optional=True,
513 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
514 "It will be removed after v20. Use colorSplitBands instead."),
515 )
516 colorSplitBands = pexConfig.ListField(
517 doc="Band names to use to split stars by color. Must have 2 entries.",
518 dtype=str,
519 length=2,
520 default=('g', 'i'),
521 )
522 modelMagErrors = pexConfig.Field(
523 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
524 dtype=bool,
525 default=True,
526 )
527 useQuadraticPwv = pexConfig.Field(
528 doc="Model PWV with a quadratic term for variation through the night?",
529 dtype=bool,
530 default=False,
531 )
532 instrumentParsPerBand = pexConfig.Field(
533 doc=("Model instrumental parameters per band? "
534 "Otherwise, instrumental parameters (QE changes with time) are "
535 "shared among all bands."),
536 dtype=bool,
537 default=False,
538 )
539 instrumentSlopeMinDeltaT = pexConfig.Field(
540 doc=("Minimum time change (in days) between observations to use in constraining "
541 "instrument slope."),
542 dtype=float,
543 default=20.0,
544 )
545 fitMirrorChromaticity = pexConfig.Field(
546 doc="Fit (intraband) mirror chromatic term?",
547 dtype=bool,
548 default=False,
549 )
550 coatingMjds = pexConfig.ListField(
551 doc="Mirror coating dates in MJD",
552 dtype=float,
553 default=(0.0,),
554 )
555 outputStandardsBeforeFinalCycle = pexConfig.Field(
556 doc="Output standard stars prior to final cycle? Used in debugging.",
557 dtype=bool,
558 default=False,
559 )
560 outputZeropointsBeforeFinalCycle = pexConfig.Field(
561 doc="Output standard stars prior to final cycle? Used in debugging.",
562 dtype=bool,
563 default=False,
564 )
565 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
566 doc=("Use star repeatability (instead of exposures) for computing photometric "
567 "cuts? Recommended for tract mode or bands with few exposures. "
568 "May be 1 element (same for all bands) or the same length as config.bands."),
569 dtype=bool,
570 default=(False,),
571 optional=True,
572 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
573 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
574 )
575 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
576 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
577 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
578 keytype=str,
579 itemtype=bool,
580 default={},
581 )
582 autoPhotometricCutNSig = pexConfig.Field(
583 doc=("Number of sigma for automatic computation of (low) photometric cut. "
584 "Cut is based on exposure gray width (per band), unless "
585 "useRepeatabilityForExpGrayCuts is set, in which case the star "
586 "repeatability is used (also per band)."),
587 dtype=float,
588 default=3.0,
589 )
590 autoHighCutNSig = pexConfig.Field(
591 doc=("Number of sigma for automatic computation of (high) outlier cut. "
592 "Cut is based on exposure gray width (per band), unless "
593 "useRepeatabilityForExpGrayCuts is set, in which case the star "
594 "repeatability is used (also per band)."),
595 dtype=float,
596 default=4.0,
597 )
598 quietMode = pexConfig.Field(
599 doc="Be less verbose with logging.",
600 dtype=bool,
601 default=False,
602 )
604 def setDefaults(self):
605 pass
607 def validate(self):
608 super().validate()
610 for band in self.fitBands:
611 if band not in self.bands:
612 msg = 'fitBand %s not in bands' % (band)
613 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
614 for band in self.requiredBands:
615 if band not in self.bands:
616 msg = 'requiredBand %s not in bands' % (band)
617 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
618 for band in self.colorSplitBands:
619 if band not in self.bands:
620 msg = 'colorSplitBand %s not in bands' % (band)
621 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
622 for band in self.bands:
623 if band not in self.superStarSubCcdDict:
624 msg = 'band %s not in superStarSubCcdDict' % (band)
625 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
626 self, msg)
627 if band not in self.ccdGraySubCcdDict:
628 msg = 'band %s not in ccdGraySubCcdDict' % (band)
629 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
630 self, msg)
631 if band not in self.expGrayPhotometricCutDict:
632 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
633 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
634 self, msg)
635 if band not in self.expGrayHighCutDict:
636 msg = 'band %s not in expGrayHighCutDict' % (band)
637 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
638 self, msg)
639 if band not in self.expVarGrayPhotometricCutDict:
640 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
641 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
642 self, msg)
643 if band not in self.sigFgcmMaxEGrayDict:
644 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
645 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
646 self, msg)
647 if band not in self.approxThroughputDict:
648 msg = 'band %s not in approxThroughputDict' % (band)
649 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
650 self, msg)
651 if band not in self.useRepeatabilityForExpGrayCutsDict:
652 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
653 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
654 self, msg)
657class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
658 """Subclass of TaskRunner for fgcmFitCycleTask
660 fgcmFitCycleTask.run() takes one argument, the butler, and uses
661 stars and visits previously extracted from dataRefs by
662 fgcmBuildStars.
663 This Runner does not perform any dataRef parallelization, but the FGCM
664 code called by the Task uses python multiprocessing (see the "ncores"
665 config option).
666 """
668 @staticmethod
669 def getTargetList(parsedCmd):
670 """
671 Return a list with one element, the butler.
672 """
673 return [parsedCmd.butler]
675 def __call__(self, butler):
676 """
677 Parameters
678 ----------
679 butler: `lsst.daf.persistence.Butler`
681 Returns
682 -------
683 exitStatus: `list` with `pipeBase.Struct`
684 exitStatus (0: success; 1: failure)
685 """
687 task = self.TaskClass(config=self.config, log=self.log)
689 exitStatus = 0
690 if self.doRaise:
691 task.runDataRef(butler)
692 else:
693 try:
694 task.runDataRef(butler)
695 except Exception as e:
696 exitStatus = 1
697 task.log.fatal("Failed: %s" % e)
698 if not isinstance(e, pipeBase.TaskError):
699 traceback.print_exc(file=sys.stderr)
701 task.writeMetadata(butler)
703 # The task does not return any results:
704 return [pipeBase.Struct(exitStatus=exitStatus)]
706 def run(self, parsedCmd):
707 """
708 Run the task, with no multiprocessing
710 Parameters
711 ----------
712 parsedCmd: ArgumentParser parsed command line
713 """
715 resultList = []
717 if self.precall(parsedCmd):
718 targetList = self.getTargetList(parsedCmd)
719 # make sure that we only get 1
720 resultList = self(targetList[0])
722 return resultList
725class FgcmFitCycleTask(pipeBase.CmdLineTask):
726 """
727 Run Single fit cycle for FGCM global calibration
728 """
730 ConfigClass = FgcmFitCycleConfig
731 RunnerClass = FgcmFitCycleRunner
732 _DefaultName = "fgcmFitCycle"
734 def __init__(self, butler=None, **kwargs):
735 """
736 Instantiate an fgcmFitCycle.
738 Parameters
739 ----------
740 butler : `lsst.daf.persistence.Butler`
741 """
743 pipeBase.CmdLineTask.__init__(self, **kwargs)
745 # no saving of metadata for now
746 def _getMetadataName(self):
747 return None
749 @pipeBase.timeMethod
750 def runDataRef(self, butler):
751 """
752 Run a single fit cycle for FGCM
754 Parameters
755 ----------
756 butler: `lsst.daf.persistence.Butler`
757 """
759 self._fgcmFitCycle(butler)
761 def writeConfig(self, butler, clobber=False, doBackup=True):
762 """Write the configuration used for processing the data, or check that an existing
763 one is equal to the new one if present. This is an override of the regular
764 version from pipe_base that knows about fgcmcycle.
766 Parameters
767 ----------
768 butler : `lsst.daf.persistence.Butler`
769 Data butler used to write the config. The config is written to dataset type
770 `CmdLineTask._getConfigName`.
771 clobber : `bool`, optional
772 A boolean flag that controls what happens if a config already has been saved:
773 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
774 - `False`: raise `TaskError` if this config does not match the existing config.
775 doBackup : `bool`, optional
776 Set to `True` to backup the config files if clobbering.
777 """
778 configName = self._getConfigName()
779 if configName is None:
780 return
781 if clobber:
782 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
783 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
784 # this may be subject to a race condition; see #2789
785 try:
786 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
787 except Exception as exc:
788 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
789 (configName, exc))
791 def logConfigMismatch(msg):
792 self.log.fatal("Comparing configuration: %s", msg)
794 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
795 raise pipeBase.TaskError(
796 ("Config does not match existing task config %r on disk; tasks configurations " +
797 "must be consistent within the same output repo (override with --clobber-config)") %
798 (configName,))
799 else:
800 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
802 def _fgcmFitCycle(self, butler):
803 """
804 Run the fit cycle
806 Parameters
807 ----------
808 butler: `lsst.daf.persistence.Butler`
809 """
811 self._checkDatasetsExist(butler)
813 # Set defaults on whether to output standards and zeropoints
814 self.maxIter = self.config.maxIterBeforeFinalCycle
815 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
816 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
817 self.resetFitParameters = True
819 if self.config.isFinalCycle:
820 # This is the final fit cycle, so we do not want to reset fit
821 # parameters, we want to run a final "clean-up" with 0 fit iterations,
822 # and we always want to output standards and zeropoints
823 self.maxIter = 0
824 self.outputStandards = True
825 self.outputZeropoints = True
826 self.resetFitParameters = False
828 camera = butler.get('camera')
829 configDict = makeConfigDict(self.config, self.log, camera,
830 self.maxIter, self.resetFitParameters,
831 self.outputZeropoints)
833 lutCat = butler.get('fgcmLookUpTable')
834 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
835 del lutCat
837 # next we need the exposure/visit information
839 # fgcmExpInfo = self._loadVisitCatalog(butler)
840 visitCat = butler.get('fgcmVisitCatalog')
841 fgcmExpInfo = translateVisitCatalog(visitCat)
842 del visitCat
844 # Use the first orientation.
845 # TODO: DM-21215 will generalize to arbitrary camera orientations
846 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
848 noFitsDict = {'lutIndex': lutIndexVals,
849 'lutStd': lutStd,
850 'expInfo': fgcmExpInfo,
851 'ccdOffsets': ccdOffsets}
853 # set up the fitter object
854 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
855 noFitsDict=noFitsDict, noOutput=True)
857 # create the parameter object
858 if (fgcmFitCycle.initialCycle):
859 # cycle = 0, initial cycle
860 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
861 fgcmLut,
862 fgcmExpInfo)
863 else:
864 inParInfo, inParams, inSuperStar = self._loadParameters(butler)
865 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
866 fgcmExpInfo,
867 inParInfo,
868 inParams,
869 inSuperStar)
871 lastCycle = configDict['cycleNumber'] - 1
873 # set up the stars...
874 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
876 starObs = butler.get('fgcmStarObservations')
877 starIds = butler.get('fgcmStarIds')
878 starIndices = butler.get('fgcmStarIndices')
880 # grab the flagged stars if available
881 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
882 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
883 flagId = flaggedStars['objId'][:]
884 flagFlag = flaggedStars['objFlag'][:]
885 else:
886 flaggedStars = None
887 flagId = None
888 flagFlag = None
890 if self.config.doReferenceCalibration:
891 refStars = butler.get('fgcmReferenceStars')
893 refMag, refMagErr = extractReferenceMags(refStars,
894 self.config.bands,
895 self.config.filterMap)
896 refId = refStars['fgcm_id'][:]
897 else:
898 refStars = None
899 refId = None
900 refMag = None
901 refMagErr = None
903 # match star observations to visits
904 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
905 # actually be transferred into fgcm using the indexing below.
906 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
908 # The fgcmStars.loadStars method will copy all the star information into
909 # special shared memory objects that will not blow up the memory usage when
910 # used with python multiprocessing. Once all the numbers are copied,
911 # it is necessary to release all references to the objects that previously
912 # stored the data to ensure that the garbage collector can clear the memory,
913 # and ensure that this memory is not copied when multiprocessing kicks in.
915 # We determine the conversion from the native units (typically radians) to
916 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
917 # numpy arrays rather than Angles, which would we approximately 600x slower.
918 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
920 fgcmStars.loadStars(fgcmPars,
921 starObs['visit'][starIndices['obsIndex']],
922 starObs['ccd'][starIndices['obsIndex']],
923 starObs['ra'][starIndices['obsIndex']] * conv,
924 starObs['dec'][starIndices['obsIndex']] * conv,
925 starObs['instMag'][starIndices['obsIndex']],
926 starObs['instMagErr'][starIndices['obsIndex']],
927 fgcmExpInfo['FILTERNAME'][visitIndex],
928 starIds['fgcm_id'][:],
929 starIds['ra'][:],
930 starIds['dec'][:],
931 starIds['obsArrIndex'][:],
932 starIds['nObs'][:],
933 obsX=starObs['x'][starIndices['obsIndex']],
934 obsY=starObs['y'][starIndices['obsIndex']],
935 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
936 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
937 refID=refId,
938 refMag=refMag,
939 refMagErr=refMagErr,
940 flagID=flagId,
941 flagFlag=flagFlag,
942 computeNobs=True)
944 # Release all references to temporary objects holding star data (see above)
945 del starObs
946 del starIds
947 del starIndices
948 del flagId
949 del flagFlag
950 del flaggedStars
951 del refStars
952 del refId
953 del refMag
954 del refMagErr
956 # and set the bits in the cycle object
957 fgcmFitCycle.setLUT(fgcmLut)
958 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
959 fgcmFitCycle.setPars(fgcmPars)
961 # finish the setup
962 fgcmFitCycle.finishSetup()
964 # and run
965 fgcmFitCycle.run()
967 ##################
968 # Persistance
969 ##################
971 self._persistFgcmDatasets(butler, fgcmFitCycle)
973 # Output the config for the next cycle
974 # We need to make a copy since the input one has been frozen
976 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
977 i, b in enumerate(self.config.bands)}
978 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
979 i, band in enumerate(self.config.bands)}
981 outConfig = copy.copy(self.config)
982 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
983 precomputeSuperStarInitialCycle=False,
984 freezeStdAtmosphere=False,
985 expGrayPhotometricCutDict=updatedPhotometricCutDict,
986 expGrayHighCutDict=updatedHighCutDict)
987 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
988 outConfig.cycleNumber)
989 outConfig.save(configFileName)
991 if self.config.isFinalCycle == 1:
992 # We are done, ready to output products
993 self.log.info("Everything is in place to run fgcmOutputProducts.py")
994 else:
995 self.log.info("Saved config for next cycle to %s" % (configFileName))
996 self.log.info("Be sure to look at:")
997 self.log.info(" config.expGrayPhotometricCut")
998 self.log.info(" config.expGrayHighCut")
999 self.log.info("If you are satisfied with the fit, please set:")
1000 self.log.info(" config.isFinalCycle = True")
1002 def _checkDatasetsExist(self, butler):
1003 """
1004 Check if necessary datasets exist to run fgcmFitCycle
1006 Parameters
1007 ----------
1008 butler: `lsst.daf.persistence.Butler`
1010 Raises
1011 ------
1012 RuntimeError
1013 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1014 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1015 If cycleNumber > 0, then also checks for fgcmFitParameters,
1016 fgcmFlaggedStars.
1017 """
1019 if not butler.datasetExists('fgcmVisitCatalog'):
1020 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1021 if not butler.datasetExists('fgcmStarObservations'):
1022 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1023 if not butler.datasetExists('fgcmStarIds'):
1024 raise RuntimeError("Could not find fgcmStarIds in repo!")
1025 if not butler.datasetExists('fgcmStarIndices'):
1026 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1027 if not butler.datasetExists('fgcmLookUpTable'):
1028 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1030 # Need additional datasets if we are not the initial cycle
1031 if (self.config.cycleNumber > 0):
1032 if not butler.datasetExists('fgcmFitParameters',
1033 fgcmcycle=self.config.cycleNumber-1):
1034 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1035 (self.config.cycleNumber-1))
1036 if not butler.datasetExists('fgcmFlaggedStars',
1037 fgcmcycle=self.config.cycleNumber-1):
1038 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1039 (self.config.cycleNumber-1))
1041 # And additional dataset if we want reference calibration
1042 if self.config.doReferenceCalibration:
1043 if not butler.datasetExists('fgcmReferenceStars'):
1044 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1045 "doReferenceCalibration is True.")
1047 def _loadParameters(self, butler):
1048 """
1049 Load FGCM parameters from a previous fit cycle
1051 Parameters
1052 ----------
1053 butler: `lsst.daf.persistence.Butler`
1055 Returns
1056 -------
1057 inParInfo: `numpy.ndarray`
1058 Numpy array parameter information formatted for input to fgcm
1059 inParameters: `numpy.ndarray`
1060 Numpy array parameter values formatted for input to fgcm
1061 inSuperStar: `numpy.array`
1062 Superstar flat formatted for input to fgcm
1063 """
1065 # note that we already checked that this is available
1066 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1068 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1069 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1071 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1072 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1073 (parLutFilterNames.size, )),
1074 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1075 ('LNTAUUNIT', 'f8'),
1076 ('LNTAUSLOPEUNIT', 'f8'),
1077 ('ALPHAUNIT', 'f8'),
1078 ('LNPWVUNIT', 'f8'),
1079 ('LNPWVSLOPEUNIT', 'f8'),
1080 ('LNPWVQUADRATICUNIT', 'f8'),
1081 ('LNPWVGLOBALUNIT', 'f8'),
1082 ('O3UNIT', 'f8'),
1083 ('QESYSUNIT', 'f8'),
1084 ('FILTEROFFSETUNIT', 'f8'),
1085 ('HASEXTERNALPWV', 'i2'),
1086 ('HASEXTERNALTAU', 'i2')])
1087 inParInfo['NCCD'] = parCat['nCcd']
1088 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1089 inParInfo['FITBANDS'][:] = parFitBands
1090 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1091 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1093 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1094 ('PARO3', 'f8', (parCat['parO3'].size, )),
1095 ('PARLNTAUINTERCEPT', 'f8',
1096 (parCat['parLnTauIntercept'].size, )),
1097 ('PARLNTAUSLOPE', 'f8',
1098 (parCat['parLnTauSlope'].size, )),
1099 ('PARLNPWVINTERCEPT', 'f8',
1100 (parCat['parLnPwvIntercept'].size, )),
1101 ('PARLNPWVSLOPE', 'f8',
1102 (parCat['parLnPwvSlope'].size, )),
1103 ('PARLNPWVQUADRATIC', 'f8',
1104 (parCat['parLnPwvQuadratic'].size, )),
1105 ('PARQESYSINTERCEPT', 'f8',
1106 (parCat['parQeSysIntercept'].size, )),
1107 ('COMPQESYSSLOPE', 'f8',
1108 (parCat['compQeSysSlope'].size, )),
1109 ('PARFILTEROFFSET', 'f8',
1110 (parCat['parFilterOffset'].size, )),
1111 ('PARFILTEROFFSETFITFLAG', 'i2',
1112 (parCat['parFilterOffsetFitFlag'].size, )),
1113 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1114 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1115 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1116 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1117 ('COMPABSTHROUGHPUT', 'f8',
1118 (parCat['compAbsThroughput'].size, )),
1119 ('COMPREFOFFSET', 'f8',
1120 (parCat['compRefOffset'].size, )),
1121 ('COMPREFSIGMA', 'f8',
1122 (parCat['compRefSigma'].size, )),
1123 ('COMPMIRRORCHROMATICITY', 'f8',
1124 (parCat['compMirrorChromaticity'].size, )),
1125 ('MIRRORCHROMATICITYPIVOT', 'f8',
1126 (parCat['mirrorChromaticityPivot'].size, )),
1127 ('COMPMEDIANSEDSLOPE', 'f8',
1128 (parCat['compMedianSedSlope'].size, )),
1129 ('COMPAPERCORRPIVOT', 'f8',
1130 (parCat['compAperCorrPivot'].size, )),
1131 ('COMPAPERCORRSLOPE', 'f8',
1132 (parCat['compAperCorrSlope'].size, )),
1133 ('COMPAPERCORRSLOPEERR', 'f8',
1134 (parCat['compAperCorrSlopeErr'].size, )),
1135 ('COMPAPERCORRRANGE', 'f8',
1136 (parCat['compAperCorrRange'].size, )),
1137 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1138 (parCat['compModelErrExptimePivot'].size, )),
1139 ('COMPMODELERRFWHMPIVOT', 'f8',
1140 (parCat['compModelErrFwhmPivot'].size, )),
1141 ('COMPMODELERRSKYPIVOT', 'f8',
1142 (parCat['compModelErrSkyPivot'].size, )),
1143 ('COMPMODELERRPARS', 'f8',
1144 (parCat['compModelErrPars'].size, )),
1145 ('COMPEXPGRAY', 'f8',
1146 (parCat['compExpGray'].size, )),
1147 ('COMPVARGRAY', 'f8',
1148 (parCat['compVarGray'].size, )),
1149 ('COMPEXPDELTAMAGBKG', 'f8',
1150 (parCat['compExpDeltaMagBkg'].size, )),
1151 ('COMPNGOODSTARPEREXP', 'i4',
1152 (parCat['compNGoodStarPerExp'].size, )),
1153 ('COMPSIGFGCM', 'f8',
1154 (parCat['compSigFgcm'].size, )),
1155 ('COMPSIGMACAL', 'f8',
1156 (parCat['compSigmaCal'].size, )),
1157 ('COMPRETRIEVEDLNPWV', 'f8',
1158 (parCat['compRetrievedLnPwv'].size, )),
1159 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1160 (parCat['compRetrievedLnPwvRaw'].size, )),
1161 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1162 (parCat['compRetrievedLnPwvFlag'].size, )),
1163 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1164 (parCat['compRetrievedTauNight'].size, ))])
1166 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1167 inParams['PARO3'][:] = parCat['parO3'][0, :]
1168 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1169 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1170 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1171 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1172 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1173 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1174 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1175 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1176 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1177 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1178 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1179 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1180 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1181 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1182 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1183 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1184 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1185 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1186 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1187 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1188 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1189 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1190 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1191 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1192 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1193 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1194 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1195 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1196 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1197 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1198 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1199 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1200 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1201 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1202 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1203 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1205 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1206 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1208 return (inParInfo, inParams, inSuperStar)
1210 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1211 """
1212 Persist FGCM datasets through the butler.
1214 Parameters
1215 ----------
1216 butler: `lsst.daf.persistence.Butler`
1217 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1218 Fgcm Fit cycle object
1219 """
1221 # Save the parameters
1222 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1224 parSchema = afwTable.Schema()
1226 comma = ','
1227 lutFilterNameString = comma.join([n.decode('utf-8')
1228 for n in parInfo['LUTFILTERNAMES'][0]])
1229 fitBandString = comma.join([n.decode('utf-8')
1230 for n in parInfo['FITBANDS'][0]])
1232 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1233 lutFilterNameString, fitBandString)
1234 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1235 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1236 lutFilterNameString, fitBandString)
1238 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1240 # Save the indices of the flagged stars
1241 # (stars that have been (a) reserved from the fit for testing and
1242 # (b) bad stars that have failed quality checks.)
1243 flagStarSchema = self._makeFlagStarSchema()
1244 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1245 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1247 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1249 # Save the zeropoint information and atmospheres only if desired
1250 if self.outputZeropoints:
1251 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1252 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1254 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1255 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1257 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1259 # Save atmosphere values
1260 # These are generated by the same code that generates zeropoints
1261 atmSchema = makeAtmSchema()
1262 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1264 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1266 # Save the standard stars (if configured)
1267 if self.outputStandards:
1268 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1269 stdSchema = makeStdSchema(len(goodBands))
1270 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1272 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1274 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1275 lutFilterNameString, fitBandString):
1276 """
1277 Make the parameter persistence schema
1279 Parameters
1280 ----------
1281 parInfo: `numpy.ndarray`
1282 Parameter information returned by fgcm
1283 pars: `numpy.ndarray`
1284 Parameter values returned by fgcm
1285 parSuperStarFlat: `numpy.array`
1286 Superstar flat values returned by fgcm
1287 lutFilterNameString: `str`
1288 Combined string of all the lutFilterNames
1289 fitBandString: `str`
1290 Combined string of all the fitBands
1292 Returns
1293 -------
1294 parSchema: `afwTable.schema`
1295 """
1297 parSchema = afwTable.Schema()
1299 # parameter info section
1300 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1301 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1302 size=len(lutFilterNameString))
1303 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1304 size=len(fitBandString))
1305 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1306 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1307 doc='Step units for ln(AOD) slope')
1308 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1309 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1310 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1311 doc='Step units for ln(pwv) slope')
1312 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1313 doc='Step units for ln(pwv) quadratic term')
1314 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1315 doc='Step units for global ln(pwv) parameters')
1316 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1317 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1318 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1319 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1320 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1322 # parameter section
1323 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1324 size=pars['PARALPHA'].size)
1325 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1326 size=pars['PARO3'].size)
1327 parSchema.addField('parLnTauIntercept', type='ArrayD',
1328 doc='ln(Tau) intercept parameter vector',
1329 size=pars['PARLNTAUINTERCEPT'].size)
1330 parSchema.addField('parLnTauSlope', type='ArrayD',
1331 doc='ln(Tau) slope parameter vector',
1332 size=pars['PARLNTAUSLOPE'].size)
1333 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1334 size=pars['PARLNPWVINTERCEPT'].size)
1335 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1336 size=pars['PARLNPWVSLOPE'].size)
1337 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1338 size=pars['PARLNPWVQUADRATIC'].size)
1339 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1340 size=pars['PARQESYSINTERCEPT'].size)
1341 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1342 size=pars[0]['COMPQESYSSLOPE'].size)
1343 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1344 size=pars['PARFILTEROFFSET'].size)
1345 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1346 size=pars['PARFILTEROFFSETFITFLAG'].size)
1347 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1348 doc='Global scale for retrieved ln(pwv)')
1349 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1350 doc='Global offset for retrieved ln(pwv)')
1351 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1352 doc='Nightly offset for retrieved ln(pwv)',
1353 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1354 parSchema.addField('compAbsThroughput', type='ArrayD',
1355 doc='Absolute throughput (relative to transmission curves)',
1356 size=pars['COMPABSTHROUGHPUT'].size)
1357 parSchema.addField('compRefOffset', type='ArrayD',
1358 doc='Offset between reference stars and calibrated stars',
1359 size=pars['COMPREFOFFSET'].size)
1360 parSchema.addField('compRefSigma', type='ArrayD',
1361 doc='Width of reference star/calibrated star distribution',
1362 size=pars['COMPREFSIGMA'].size)
1363 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1364 doc='Computed mirror chromaticity terms',
1365 size=pars['COMPMIRRORCHROMATICITY'].size)
1366 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1367 doc='Mirror chromaticity pivot mjd',
1368 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1369 parSchema.addField('compMedianSedSlope', type='ArrayD',
1370 doc='Computed median SED slope (per band)',
1371 size=pars['COMPMEDIANSEDSLOPE'].size)
1372 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1373 size=pars['COMPAPERCORRPIVOT'].size)
1374 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1375 size=pars['COMPAPERCORRSLOPE'].size)
1376 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1377 size=pars['COMPAPERCORRSLOPEERR'].size)
1378 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1379 size=pars['COMPAPERCORRRANGE'].size)
1380 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1381 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1382 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1383 size=pars['COMPMODELERRFWHMPIVOT'].size)
1384 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1385 size=pars['COMPMODELERRSKYPIVOT'].size)
1386 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1387 size=pars['COMPMODELERRPARS'].size)
1388 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1389 size=pars['COMPEXPGRAY'].size)
1390 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1391 size=pars['COMPVARGRAY'].size)
1392 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1393 doc='Computed exposure offset due to background',
1394 size=pars['COMPEXPDELTAMAGBKG'].size)
1395 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1396 doc='Computed number of good stars per exposure',
1397 size=pars['COMPNGOODSTARPEREXP'].size)
1398 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1399 size=pars['COMPSIGFGCM'].size)
1400 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1401 size=pars['COMPSIGMACAL'].size)
1402 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1403 size=pars['COMPRETRIEVEDLNPWV'].size)
1404 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1405 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1406 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1407 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1408 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1409 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1410 # superstarflat section
1411 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1412 size=4)
1413 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1414 size=parSuperStarFlat.size)
1416 return parSchema
1418 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1419 lutFilterNameString, fitBandString):
1420 """
1421 Make the FGCM parameter catalog for persistence
1423 Parameters
1424 ----------
1425 parSchema: `lsst.afw.table.Schema`
1426 Parameter catalog schema
1427 pars: `numpy.ndarray`
1428 FGCM parameters to put into parCat
1429 parSuperStarFlat: `numpy.array`
1430 FGCM superstar flat array to put into parCat
1431 lutFilterNameString: `str`
1432 Combined string of all the lutFilterNames
1433 fitBandString: `str`
1434 Combined string of all the fitBands
1436 Returns
1437 -------
1438 parCat: `afwTable.BasicCatalog`
1439 Atmosphere and instrumental model parameter catalog for persistence
1440 """
1442 parCat = afwTable.BaseCatalog(parSchema)
1443 parCat.reserve(1)
1445 # The parameter catalog just has one row, with many columns for all the
1446 # atmosphere and instrument fit parameters
1447 rec = parCat.addNew()
1449 # info section
1450 rec['nCcd'] = parInfo['NCCD']
1451 rec['lutFilterNames'] = lutFilterNameString
1452 rec['fitBands'] = fitBandString
1453 # note these are not currently supported here.
1454 rec['hasExternalPwv'] = 0
1455 rec['hasExternalTau'] = 0
1457 # parameter section
1459 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1461 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1462 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1463 'parQeSysIntercept', 'compQeSysSlope',
1464 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1465 'parFilterOffset', 'parFilterOffsetFitFlag',
1466 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1467 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1468 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1469 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1470 'compModelErrSkyPivot', 'compModelErrPars',
1471 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1472 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1473 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1474 'compRetrievedTauNight']
1476 for scalarName in scalarNames:
1477 rec[scalarName] = pars[scalarName.upper()]
1479 for arrName in arrNames:
1480 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1482 # superstar section
1483 rec['superstarSize'][:] = parSuperStarFlat.shape
1484 rec['superstar'][:] = parSuperStarFlat.flatten()
1486 return parCat
1488 def _makeFlagStarSchema(self):
1489 """
1490 Make the flagged-stars schema
1492 Returns
1493 -------
1494 flagStarSchema: `lsst.afw.table.Schema`
1495 """
1497 flagStarSchema = afwTable.Schema()
1499 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1500 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1502 return flagStarSchema
1504 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1505 """
1506 Make the flagged star catalog for persistence
1508 Parameters
1509 ----------
1510 flagStarSchema: `lsst.afw.table.Schema`
1511 Flagged star schema
1512 flagStarStruct: `numpy.ndarray`
1513 Flagged star structure from fgcm
1515 Returns
1516 -------
1517 flagStarCat: `lsst.afw.table.BaseCatalog`
1518 Flagged star catalog for persistence
1519 """
1521 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1522 flagStarCat.resize(flagStarStruct.size)
1524 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1525 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1527 return flagStarCat