lsst.fgcmcal  19.0.0-18-gef3cd1a+2
fgcmFitCycle.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Perform a single fit cycle of FGCM.
24 
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
28 for the global fit).
29 
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
34 """
35 
36 import sys
37 import traceback
38 import copy
39 
40 import numpy as np
41 
42 import lsst.pex.config as pexConfig
43 import lsst.pipe.base as pipeBase
44 import lsst.afw.table as afwTable
45 
46 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47 from .utilities import extractReferenceMags
48 from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49 from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50 from .sedterms import SedboundarytermDict, SedtermDict
51 
52 import fgcm
53 
54 __all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
55 
56 
57 class FgcmFitCycleConfig(pexConfig.Config):
58  """Config for FgcmFitCycle"""
59 
60  bands = pexConfig.ListField(
61  doc="Bands to run calibration",
62  dtype=str,
63  default=[],
64  )
65  fitFlag = pexConfig.ListField(
66  doc=("Flag for which bands are directly constrained in the FGCM fit. "
67  "Bands set to 0 will have the atmosphere constrained from observations "
68  "in other bands on the same night. Must be same length as config.bands, "
69  "and matched band-by-band."),
70  dtype=int,
71  default=(0,),
72  optional=True,
73  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
74  "It will be removed after v20. Use fitBands instead."),
75  )
76  fitBands = pexConfig.ListField(
77  doc=("Bands to use in atmospheric fit. The bands not listed here will have "
78  "the atmosphere constrained from the 'fitBands' on the same night. "
79  "Must be a subset of `config.bands`"),
80  dtype=str,
81  default=[],
82  )
83  requiredFlag = pexConfig.ListField(
84  doc=("Flag for which bands are required for a star to be considered a calibration "
85  "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86  "be same length as config.bands, and matched band-by-band."),
87  dtype=int,
88  default=(0,),
89  optional=True,
90  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
91  "It will be removed after v20. Use requiredBands instead."),
92  )
93  requiredBands = pexConfig.ListField(
94  doc=("Bands that are required for a star to be considered a calibration star. "
95  "Must be a subset of `config.bands`"),
96  dtype=str,
97  default=[],
98  )
99  filterMap = pexConfig.DictField(
100  doc="Mapping from 'filterName' to band.",
101  keytype=str,
102  itemtype=str,
103  default={},
104  )
105  doReferenceCalibration = pexConfig.Field(
106  doc="Use reference catalog as additional constraint on calibration",
107  dtype=bool,
108  default=True,
109  )
110  refStarSnMin = pexConfig.Field(
111  doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
112  dtype=float,
113  default=50.0,
114  )
115  refStarOutlierNSig = pexConfig.Field(
116  doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
117  "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
118  dtype=float,
119  default=4.0,
120  )
121  applyRefStarColorCuts = pexConfig.Field(
122  doc="Apply color cuts to reference stars?",
123  dtype=bool,
124  default=True,
125  )
126  nCore = pexConfig.Field(
127  doc="Number of cores to use",
128  dtype=int,
129  default=4,
130  )
131  nStarPerRun = pexConfig.Field(
132  doc="Number of stars to run in each chunk",
133  dtype=int,
134  default=200000,
135  )
136  nExpPerRun = pexConfig.Field(
137  doc="Number of exposures to run in each chunk",
138  dtype=int,
139  default=1000,
140  )
141  reserveFraction = pexConfig.Field(
142  doc="Fraction of stars to reserve for testing",
143  dtype=float,
144  default=0.1,
145  )
146  freezeStdAtmosphere = pexConfig.Field(
147  doc="Freeze atmosphere parameters to standard (for testing)",
148  dtype=bool,
149  default=False,
150  )
151  precomputeSuperStarInitialCycle = pexConfig.Field(
152  doc="Precompute superstar flat for initial cycle",
153  dtype=bool,
154  default=False,
155  )
156  superStarSubCcd = pexConfig.Field(
157  doc="Compute superstar flat on sub-ccd scale",
158  dtype=bool,
159  default=True,
160  optional=True,
161  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
162  "It will be removed after v20. Use superStarSubCcdDict instead."),
163  )
164  superStarSubCcdDict = pexConfig.DictField(
165  doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166  "Must have one entry per band."),
167  keytype=str,
168  itemtype=bool,
169  default={},
170  )
171  superStarSubCcdChebyshevOrder = pexConfig.Field(
172  doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173  "Global default is first-order polynomials, and should be overridden "
174  "on a camera-by-camera basis depending on the ISR."),
175  dtype=int,
176  default=1,
177  )
178  superStarSubCcdTriangular = pexConfig.Field(
179  doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
180  "suppress high-order cross terms?"),
181  dtype=bool,
182  default=False,
183  )
184  superStarSigmaClip = pexConfig.Field(
185  doc="Number of sigma to clip outliers when selecting for superstar flats",
186  dtype=float,
187  default=5.0,
188  )
189  ccdGraySubCcd = pexConfig.Field(
190  doc="Compute CCD gray terms on sub-ccd scale",
191  dtype=bool,
192  default=False,
193  optional=True,
194  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
195  "It will be removed after v20. Use ccdGraySubCcdDict instead."),
196  )
197  ccdGraySubCcdDict = pexConfig.DictField(
198  doc=("Per-band specification on whether to compute achromatic per-ccd residual "
199  "('ccd gray') on a sub-ccd scale."),
200  keytype=str,
201  itemtype=bool,
202  default={},
203  )
204  ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205  doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
206  dtype=int,
207  default=1,
208  )
209  ccdGraySubCcdTriangular = pexConfig.Field(
210  doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
211  "suppress high-order cross terms?"),
212  dtype=bool,
213  default=True,
214  )
215  cycleNumber = pexConfig.Field(
216  doc=("FGCM fit cycle number. This is automatically incremented after each run "
217  "and stage of outlier rejection. See cookbook for details."),
218  dtype=int,
219  default=None,
220  )
221  isFinalCycle = pexConfig.Field(
222  doc=("Is this the final cycle of the fitting? Will automatically compute final "
223  "selection of stars and photometric exposures, and will output zeropoints "
224  "and standard stars for use in fgcmOutputProducts"),
225  dtype=bool,
226  default=False,
227  )
228  maxIterBeforeFinalCycle = pexConfig.Field(
229  doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
230  "will always be 0 in the final cycle for cleanup and final selection."),
231  dtype=int,
232  default=50,
233  )
234  utBoundary = pexConfig.Field(
235  doc="Boundary (in UTC) from day-to-day",
236  dtype=float,
237  default=None,
238  )
239  washMjds = pexConfig.ListField(
240  doc="Mirror wash MJDs",
241  dtype=float,
242  default=(0.0,),
243  )
244  epochMjds = pexConfig.ListField(
245  doc="Epoch boundaries in MJD",
246  dtype=float,
247  default=(0.0,),
248  )
249  minObsPerBand = pexConfig.Field(
250  doc="Minimum good observations per band",
251  dtype=int,
252  default=2,
253  )
254  # TODO: When DM-16511 is done, it will be possible to get the
255  # telescope latitude directly from the camera.
256  latitude = pexConfig.Field(
257  doc="Observatory latitude",
258  dtype=float,
259  default=None,
260  )
261  pixelScale = pexConfig.Field(
262  doc="Pixel scale (arcsec/pixel) (temporary)",
263  dtype=float,
264  deprecated=("This field is no longer used, and has been deprecated by DM-16490. "
265  "It will be removed after v19."),
266  optional=True,
267  )
268  brightObsGrayMax = pexConfig.Field(
269  doc="Maximum gray extinction to be considered bright observation",
270  dtype=float,
271  default=0.15,
272  )
273  minStarPerCcd = pexConfig.Field(
274  doc=("Minimum number of good stars per CCD to be used in calibration fit. "
275  "CCDs with fewer stars will have their calibration estimated from other "
276  "CCDs in the same visit, with zeropoint error increased accordingly."),
277  dtype=int,
278  default=5,
279  )
280  minCcdPerExp = pexConfig.Field(
281  doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
282  "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
283  dtype=int,
284  default=5,
285  )
286  maxCcdGrayErr = pexConfig.Field(
287  doc="Maximum error on CCD gray offset to be considered photometric",
288  dtype=float,
289  default=0.05,
290  )
291  minStarPerExp = pexConfig.Field(
292  doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
293  "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
294  dtype=int,
295  default=600,
296  )
297  minExpPerNight = pexConfig.Field(
298  doc="Minimum number of good exposures/visits to consider a partly photometric night",
299  dtype=int,
300  default=10,
301  )
302  expGrayInitialCut = pexConfig.Field(
303  doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
304  "observations."),
305  dtype=float,
306  default=-0.25,
307  )
308  expGrayPhotometricCut = pexConfig.ListField(
309  doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
310  "Must be same length as config.bands, and matched band-by-band."),
311  dtype=float,
312  default=(0.0,),
313  optional=True,
314  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
315  "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
316  )
317  expGrayPhotometricCutDict = pexConfig.DictField(
318  doc=("Per-band specification on maximum (negative) achromatic exposure residual "
319  "('gray term') for a visit to be considered photometric. Must have one "
320  "entry per band. Broad-band filters should be -0.05."),
321  keytype=str,
322  itemtype=float,
323  default={},
324  )
325  expGrayHighCut = pexConfig.ListField(
326  doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
327  "Must be same length as config.bands, and matched band-by-band."),
328  dtype=float,
329  default=(0.0,),
330  optional=True,
331  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
332  "It will be removed after v20. Use expGrayHighCutDict instead."),
333  )
334  expGrayHighCutDict = pexConfig.DictField(
335  doc=("Per-band specification on maximum (positive) achromatic exposure residual "
336  "('gray term') for a visit to be considered photometric. Must have one "
337  "entry per band. Broad-band filters should be 0.2."),
338  keytype=str,
339  itemtype=float,
340  default={},
341  )
342  expGrayRecoverCut = pexConfig.Field(
343  doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
344  "Visits with more gray extinction will only get CCD zeropoints if there are "
345  "sufficient star observations (minStarPerCcd) on that CCD."),
346  dtype=float,
347  default=-1.0,
348  )
349  expVarGrayPhotometricCut = pexConfig.Field(
350  doc="Maximum exposure variance to be considered possibly photometric",
351  dtype=float,
352  default=0.0005,
353  optional=True,
354  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
355  "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
356  )
357  expVarGrayPhotometricCutDict = pexConfig.DictField(
358  doc=("Per-band specification on maximum exposure variance to be considered possibly "
359  "photometric. Must have one entry per band. Broad-band filters should be "
360  "0.0005."),
361  keytype=str,
362  itemtype=float,
363  default={},
364  )
365  expGrayErrRecoverCut = pexConfig.Field(
366  doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
367  "Visits with more gray variance will only get CCD zeropoints if there are "
368  "sufficient star observations (minStarPerCcd) on that CCD."),
369  dtype=float,
370  default=0.05,
371  )
372  aperCorrFitNBins = pexConfig.Field(
373  doc=("Number of aperture bins used in aperture correction fit. When set to 0"
374  "no fit will be performed, and the config.aperCorrInputSlopes will be "
375  "used if available."),
376  dtype=int,
377  default=10,
378  )
379  aperCorrInputSlopes = pexConfig.ListField(
380  doc=("Aperture correction input slope parameters. These are used on the first "
381  "fit iteration, and aperture correction parameters will be updated from "
382  "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
383  "when there is insufficient data to fit the parameters (e.g. tract mode). "
384  "If set, must be same length as config.bands, and matched band-by-band."),
385  dtype=float,
386  default=[],
387  optional=True,
388  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
389  "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
390  )
391  aperCorrInputSlopeDict = pexConfig.DictField(
392  doc=("Per-band specification of aperture correction input slope parameters. These "
393  "are used on the first fit iteration, and aperture correction parameters will "
394  "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
395  "to set this when there is insufficient data to fit the parameters (e.g. "
396  "tract mode)."),
397  keytype=str,
398  itemtype=float,
399  default={},
400  )
401  sedFudgeFactors = pexConfig.ListField(
402  doc=("Fudge factors for computing linear SED from colors. Must be same length as "
403  "config.bands, and matched band-by-band."),
404  dtype=float,
405  default=(0,),
406  optional=True,
407  deprecated=("This field has been deprecated and will be removed after v20. "
408  "Please use sedSlopeTermMap and sedSlopeMap."),
409  )
410  sedboundaryterms = pexConfig.ConfigField(
411  doc="Mapping from bands to SED boundary term names used is sedterms.",
412  dtype=SedboundarytermDict,
413  )
414  sedterms = pexConfig.ConfigField(
415  doc="Mapping from terms to bands for fgcm linear SED approximations.",
416  dtype=SedtermDict,
417  )
418  sigFgcmMaxErr = pexConfig.Field(
419  doc="Maximum mag error for fitting sigma_FGCM",
420  dtype=float,
421  default=0.01,
422  )
423  sigFgcmMaxEGray = pexConfig.ListField(
424  doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
425  "May be 1 element (same for all bands) or the same length as config.bands."),
426  dtype=float,
427  default=(0.05,),
428  optional=True,
429  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
430  "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
431  )
432  sigFgcmMaxEGrayDict = pexConfig.DictField(
433  doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
434  "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
435  "should be 0.05."),
436  keytype=str,
437  itemtype=float,
438  default={},
439  )
440  ccdGrayMaxStarErr = pexConfig.Field(
441  doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
442  "computation"),
443  dtype=float,
444  default=0.10,
445  )
446  approxThroughput = pexConfig.ListField(
447  doc=("Approximate overall throughput at start of calibration observations. "
448  "May be 1 element (same for all bands) or the same length as config.bands."),
449  dtype=float,
450  default=(1.0, ),
451  optional=True,
452  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
453  "It will be removed after v20. Use approxThroughputDict instead."),
454  )
455  approxThroughputDict = pexConfig.DictField(
456  doc=("Per-band specification of the approximate overall throughput at the start of "
457  "calibration observations. Must have one entry per band. Typically should "
458  "be 1.0."),
459  keytype=str,
460  itemtype=float,
461  default={},
462  )
463  sigmaCalRange = pexConfig.ListField(
464  doc="Allowed range for systematic error floor estimation",
465  dtype=float,
466  default=(0.001, 0.003),
467  )
468  sigmaCalFitPercentile = pexConfig.ListField(
469  doc="Magnitude percentile range to fit systematic error floor",
470  dtype=float,
471  default=(0.05, 0.15),
472  )
473  sigmaCalPlotPercentile = pexConfig.ListField(
474  doc="Magnitude percentile range to plot systematic error floor",
475  dtype=float,
476  default=(0.05, 0.95),
477  )
478  sigma0Phot = pexConfig.Field(
479  doc="Systematic error floor for all zeropoints",
480  dtype=float,
481  default=0.003,
482  )
483  mapLongitudeRef = pexConfig.Field(
484  doc="Reference longitude for plotting maps",
485  dtype=float,
486  default=0.0,
487  )
488  mapNSide = pexConfig.Field(
489  doc="Healpix nside for plotting maps",
490  dtype=int,
491  default=256,
492  )
493  outfileBase = pexConfig.Field(
494  doc="Filename start for plot output files",
495  dtype=str,
496  default=None,
497  )
498  starColorCuts = pexConfig.ListField(
499  doc="Encoded star-color cuts (to be cleaned up)",
500  dtype=str,
501  default=("NO_DATA",),
502  )
503  colorSplitIndices = pexConfig.ListField(
504  doc="Band indices to use to split stars by color",
505  dtype=int,
506  default=None,
507  optional=True,
508  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
509  "It will be removed after v20. Use colorSplitBands instead."),
510  )
511  colorSplitBands = pexConfig.ListField(
512  doc="Band names to use to split stars by color. Must have 2 entries.",
513  dtype=str,
514  length=2,
515  default=('g', 'i'),
516  )
517  modelMagErrors = pexConfig.Field(
518  doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
519  dtype=bool,
520  default=True,
521  )
522  useQuadraticPwv = pexConfig.Field(
523  doc="Model PWV with a quadratic term for variation through the night?",
524  dtype=bool,
525  default=False,
526  )
527  instrumentParsPerBand = pexConfig.Field(
528  doc=("Model instrumental parameters per band? "
529  "Otherwise, instrumental parameters (QE changes with time) are "
530  "shared among all bands."),
531  dtype=bool,
532  default=False,
533  )
534  instrumentSlopeMinDeltaT = pexConfig.Field(
535  doc=("Minimum time change (in days) between observations to use in constraining "
536  "instrument slope."),
537  dtype=float,
538  default=20.0,
539  )
540  fitMirrorChromaticity = pexConfig.Field(
541  doc="Fit (intraband) mirror chromatic term?",
542  dtype=bool,
543  default=False,
544  )
545  coatingMjds = pexConfig.ListField(
546  doc="Mirror coating dates in MJD",
547  dtype=float,
548  default=(0.0,),
549  )
550  outputStandardsBeforeFinalCycle = pexConfig.Field(
551  doc="Output standard stars prior to final cycle? Used in debugging.",
552  dtype=bool,
553  default=False,
554  )
555  outputZeropointsBeforeFinalCycle = pexConfig.Field(
556  doc="Output standard stars prior to final cycle? Used in debugging.",
557  dtype=bool,
558  default=False,
559  )
560  useRepeatabilityForExpGrayCuts = pexConfig.ListField(
561  doc=("Use star repeatability (instead of exposures) for computing photometric "
562  "cuts? Recommended for tract mode or bands with few exposures. "
563  "May be 1 element (same for all bands) or the same length as config.bands."),
564  dtype=bool,
565  default=(False,),
566  optional=True,
567  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
568  "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
569  )
570  useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
571  doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
572  "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
573  keytype=str,
574  itemtype=bool,
575  default={},
576  )
577  autoPhotometricCutNSig = pexConfig.Field(
578  doc=("Number of sigma for automatic computation of (low) photometric cut. "
579  "Cut is based on exposure gray width (per band), unless "
580  "useRepeatabilityForExpGrayCuts is set, in which case the star "
581  "repeatability is used (also per band)."),
582  dtype=float,
583  default=3.0,
584  )
585  autoHighCutNSig = pexConfig.Field(
586  doc=("Number of sigma for automatic computation of (high) outlier cut. "
587  "Cut is based on exposure gray width (per band), unless "
588  "useRepeatabilityForExpGrayCuts is set, in which case the star "
589  "repeatability is used (also per band)."),
590  dtype=float,
591  default=4.0,
592  )
593  quietMode = pexConfig.Field(
594  doc="Be less verbose with logging.",
595  dtype=bool,
596  default=False,
597  )
598 
599  def setDefaults(self):
600  pass
601 
602  def validate(self):
603  super().validate()
604 
605  for band in self.fitBands:
606  if band not in self.bands:
607  msg = 'fitBand %s not in bands' % (band)
608  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
609  for band in self.requiredBands:
610  if band not in self.bands:
611  msg = 'requiredBand %s not in bands' % (band)
612  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
613  for band in self.colorSplitBands:
614  if band not in self.bands:
615  msg = 'colorSplitBand %s not in bands' % (band)
616  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
617  for band in self.bands:
618  if band not in self.superStarSubCcdDict:
619  msg = 'band %s not in superStarSubCcdDict' % (band)
620  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
621  self, msg)
622  if band not in self.ccdGraySubCcdDict:
623  msg = 'band %s not in ccdGraySubCcdDict' % (band)
624  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
625  self, msg)
626  if band not in self.expGrayPhotometricCutDict:
627  msg = 'band %s not in expGrayPhotometricCutDict' % (band)
628  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
629  self, msg)
630  if band not in self.expGrayHighCutDict:
631  msg = 'band %s not in expGrayHighCutDict' % (band)
632  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
633  self, msg)
634  if band not in self.expVarGrayPhotometricCutDict:
635  msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
636  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
637  self, msg)
638  if band not in self.sigFgcmMaxEGrayDict:
639  msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
640  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
641  self, msg)
642  if band not in self.approxThroughputDict:
643  msg = 'band %s not in approxThroughputDict' % (band)
644  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
645  self, msg)
646  if band not in self.useRepeatabilityForExpGrayCutsDict:
647  msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
648  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
649  self, msg)
650 
651 
652 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
653  """Subclass of TaskRunner for fgcmFitCycleTask
654 
655  fgcmFitCycleTask.run() takes one argument, the butler, and uses
656  stars and visits previously extracted from dataRefs by
657  fgcmBuildStars.
658  This Runner does not perform any dataRef parallelization, but the FGCM
659  code called by the Task uses python multiprocessing (see the "ncores"
660  config option).
661  """
662 
663  @staticmethod
664  def getTargetList(parsedCmd):
665  """
666  Return a list with one element, the butler.
667  """
668  return [parsedCmd.butler]
669 
670  def __call__(self, butler):
671  """
672  Parameters
673  ----------
674  butler: `lsst.daf.persistence.Butler`
675 
676  Returns
677  -------
678  exitStatus: `list` with `pipeBase.Struct`
679  exitStatus (0: success; 1: failure)
680  """
681 
682  task = self.TaskClass(config=self.config, log=self.log)
683 
684  exitStatus = 0
685  if self.doRaise:
686  task.runDataRef(butler)
687  else:
688  try:
689  task.runDataRef(butler)
690  except Exception as e:
691  exitStatus = 1
692  task.log.fatal("Failed: %s" % e)
693  if not isinstance(e, pipeBase.TaskError):
694  traceback.print_exc(file=sys.stderr)
695 
696  task.writeMetadata(butler)
697 
698  # The task does not return any results:
699  return [pipeBase.Struct(exitStatus=exitStatus)]
700 
701  def run(self, parsedCmd):
702  """
703  Run the task, with no multiprocessing
704 
705  Parameters
706  ----------
707  parsedCmd: ArgumentParser parsed command line
708  """
709 
710  resultList = []
711 
712  if self.precall(parsedCmd):
713  targetList = self.getTargetList(parsedCmd)
714  # make sure that we only get 1
715  resultList = self(targetList[0])
716 
717  return resultList
718 
719 
720 class FgcmFitCycleTask(pipeBase.CmdLineTask):
721  """
722  Run Single fit cycle for FGCM global calibration
723  """
724 
725  ConfigClass = FgcmFitCycleConfig
726  RunnerClass = FgcmFitCycleRunner
727  _DefaultName = "fgcmFitCycle"
728 
729  def __init__(self, butler=None, **kwargs):
730  """
731  Instantiate an fgcmFitCycle.
732 
733  Parameters
734  ----------
735  butler : `lsst.daf.persistence.Butler`
736  """
737 
738  pipeBase.CmdLineTask.__init__(self, **kwargs)
739 
740  # no saving of metadata for now
741  def _getMetadataName(self):
742  return None
743 
744  @pipeBase.timeMethod
745  def runDataRef(self, butler):
746  """
747  Run a single fit cycle for FGCM
748 
749  Parameters
750  ----------
751  butler: `lsst.daf.persistence.Butler`
752  """
753 
754  self._fgcmFitCycle(butler)
755 
756  def writeConfig(self, butler, clobber=False, doBackup=True):
757  """Write the configuration used for processing the data, or check that an existing
758  one is equal to the new one if present. This is an override of the regular
759  version from pipe_base that knows about fgcmcycle.
760 
761  Parameters
762  ----------
763  butler : `lsst.daf.persistence.Butler`
764  Data butler used to write the config. The config is written to dataset type
765  `CmdLineTask._getConfigName`.
766  clobber : `bool`, optional
767  A boolean flag that controls what happens if a config already has been saved:
768  - `True`: overwrite or rename the existing config, depending on ``doBackup``.
769  - `False`: raise `TaskError` if this config does not match the existing config.
770  doBackup : `bool`, optional
771  Set to `True` to backup the config files if clobbering.
772  """
773  configName = self._getConfigName()
774  if configName is None:
775  return
776  if clobber:
777  butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
778  elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
779  # this may be subject to a race condition; see #2789
780  try:
781  oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
782  except Exception as exc:
783  raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
784  (configName, exc))
785 
786  def logConfigMismatch(msg):
787  self.log.fatal("Comparing configuration: %s", msg)
788 
789  if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
790  raise pipeBase.TaskError(
791  ("Config does not match existing task config %r on disk; tasks configurations " +
792  "must be consistent within the same output repo (override with --clobber-config)") %
793  (configName,))
794  else:
795  butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
796 
797  def _fgcmFitCycle(self, butler):
798  """
799  Run the fit cycle
800 
801  Parameters
802  ----------
803  butler: `lsst.daf.persistence.Butler`
804  """
805 
806  self._checkDatasetsExist(butler)
807 
808  # Set defaults on whether to output standards and zeropoints
809  self.maxIter = self.config.maxIterBeforeFinalCycle
810  self.outputStandards = self.config.outputStandardsBeforeFinalCycle
811  self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
812  self.resetFitParameters = True
813 
814  if self.config.isFinalCycle:
815  # This is the final fit cycle, so we do not want to reset fit
816  # parameters, we want to run a final "clean-up" with 0 fit iterations,
817  # and we always want to output standards and zeropoints
818  self.maxIter = 0
819  self.outputStandards = True
820  self.outputZeropoints = True
821  self.resetFitParameters = False
822 
823  camera = butler.get('camera')
824  configDict = makeConfigDict(self.config, self.log, camera,
825  self.maxIter, self.resetFitParameters,
826  self.outputZeropoints)
827 
828  lutCat = butler.get('fgcmLookUpTable')
829  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
830  del lutCat
831 
832  # next we need the exposure/visit information
833 
834  # fgcmExpInfo = self._loadVisitCatalog(butler)
835  visitCat = butler.get('fgcmVisitCatalog')
836  fgcmExpInfo = translateVisitCatalog(visitCat)
837  del visitCat
838 
839  # Use the first orientation.
840  # TODO: DM-21215 will generalize to arbitrary camera orientations
841  ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
842 
843  noFitsDict = {'lutIndex': lutIndexVals,
844  'lutStd': lutStd,
845  'expInfo': fgcmExpInfo,
846  'ccdOffsets': ccdOffsets}
847 
848  # set up the fitter object
849  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
850  noFitsDict=noFitsDict, noOutput=True)
851 
852  # create the parameter object
853  if (fgcmFitCycle.initialCycle):
854  # cycle = 0, initial cycle
855  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
856  fgcmLut,
857  fgcmExpInfo)
858  else:
859  inParInfo, inParams, inSuperStar = self._loadParameters(butler)
860  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
861  fgcmExpInfo,
862  inParInfo,
863  inParams,
864  inSuperStar)
865 
866  lastCycle = configDict['cycleNumber'] - 1
867 
868  # set up the stars...
869  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
870 
871  starObs = butler.get('fgcmStarObservations')
872  starIds = butler.get('fgcmStarIds')
873  starIndices = butler.get('fgcmStarIndices')
874 
875  # grab the flagged stars if available
876  if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
877  flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
878  flagId = flaggedStars['objId'][:]
879  flagFlag = flaggedStars['objFlag'][:]
880  else:
881  flagId = None
882  flagFlag = None
883 
884  if self.config.doReferenceCalibration:
885  refStars = butler.get('fgcmReferenceStars')
886 
887  refMag, refMagErr = extractReferenceMags(refStars,
888  self.config.bands,
889  self.config.filterMap)
890  refId = refStars['fgcm_id'][:]
891  else:
892  refId = None
893  refMag = None
894  refMagErr = None
895 
896  # match star observations to visits
897  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
898  # actually be transferred into fgcm using the indexing below.
899  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
900 
901  # The fgcmStars.loadStars method will copy all the star information into
902  # special shared memory objects that will not blow up the memory usage when
903  # used with python multiprocessing. Once all the numbers are copied,
904  # it is necessary to release all references to the objects that previously
905  # stored the data to ensure that the garbage collector can clear the memory,
906  # and ensure that this memory is not copied when multiprocessing kicks in.
907 
908  # We determine the conversion from the native units (typically radians) to
909  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
910  # numpy arrays rather than Angles, which would we approximately 600x slower.
911  conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
912 
913  fgcmStars.loadStars(fgcmPars,
914  starObs['visit'][starIndices['obsIndex']],
915  starObs['ccd'][starIndices['obsIndex']],
916  starObs['ra'][starIndices['obsIndex']] * conv,
917  starObs['dec'][starIndices['obsIndex']] * conv,
918  starObs['instMag'][starIndices['obsIndex']],
919  starObs['instMagErr'][starIndices['obsIndex']],
920  fgcmExpInfo['FILTERNAME'][visitIndex],
921  starIds['fgcm_id'][:],
922  starIds['ra'][:],
923  starIds['dec'][:],
924  starIds['obsArrIndex'][:],
925  starIds['nObs'][:],
926  obsX=starObs['x'][starIndices['obsIndex']],
927  obsY=starObs['y'][starIndices['obsIndex']],
928  psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
929  refID=refId,
930  refMag=refMag,
931  refMagErr=refMagErr,
932  flagID=flagId,
933  flagFlag=flagFlag,
934  computeNobs=True)
935 
936  # Release all references to temporary objects holding star data (see above)
937  starObs = None
938  starIds = None
939  starIndices = None
940  flagId = None
941  flagFlag = None
942  flaggedStars = None
943  refStars = None
944 
945  # and set the bits in the cycle object
946  fgcmFitCycle.setLUT(fgcmLut)
947  fgcmFitCycle.setStars(fgcmStars)
948  fgcmFitCycle.setPars(fgcmPars)
949 
950  # finish the setup
951  fgcmFitCycle.finishSetup()
952 
953  # and run
954  fgcmFitCycle.run()
955 
956 
959 
960  self._persistFgcmDatasets(butler, fgcmFitCycle)
961 
962  # Output the config for the next cycle
963  # We need to make a copy since the input one has been frozen
964 
965  updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
966  i, b in enumerate(self.config.bands)}
967  updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
968  i, band in enumerate(self.config.bands)}
969 
970  outConfig = copy.copy(self.config)
971  outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
972  precomputeSuperStarInitialCycle=False,
973  freezeStdAtmosphere=False,
974  expGrayPhotometricCutDict=updatedPhotometricCutDict,
975  expGrayHighCutDict=updatedHighCutDict)
976  configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
977  outConfig.cycleNumber)
978  outConfig.save(configFileName)
979 
980  if self.config.isFinalCycle == 1:
981  # We are done, ready to output products
982  self.log.info("Everything is in place to run fgcmOutputProducts.py")
983  else:
984  self.log.info("Saved config for next cycle to %s" % (configFileName))
985  self.log.info("Be sure to look at:")
986  self.log.info(" config.expGrayPhotometricCut")
987  self.log.info(" config.expGrayHighCut")
988  self.log.info("If you are satisfied with the fit, please set:")
989  self.log.info(" config.isFinalCycle = True")
990 
991  def _checkDatasetsExist(self, butler):
992  """
993  Check if necessary datasets exist to run fgcmFitCycle
994 
995  Parameters
996  ----------
997  butler: `lsst.daf.persistence.Butler`
998 
999  Raises
1000  ------
1001  RuntimeError
1002  If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1003  fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1004  If cycleNumber > 0, then also checks for fgcmFitParameters,
1005  fgcmFlaggedStars.
1006  """
1007 
1008  if not butler.datasetExists('fgcmVisitCatalog'):
1009  raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1010  if not butler.datasetExists('fgcmStarObservations'):
1011  raise RuntimeError("Could not find fgcmStarObservations in repo!")
1012  if not butler.datasetExists('fgcmStarIds'):
1013  raise RuntimeError("Could not find fgcmStarIds in repo!")
1014  if not butler.datasetExists('fgcmStarIndices'):
1015  raise RuntimeError("Could not find fgcmStarIndices in repo!")
1016  if not butler.datasetExists('fgcmLookUpTable'):
1017  raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1018 
1019  # Need additional datasets if we are not the initial cycle
1020  if (self.config.cycleNumber > 0):
1021  if not butler.datasetExists('fgcmFitParameters',
1022  fgcmcycle=self.config.cycleNumber-1):
1023  raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1024  (self.config.cycleNumber-1))
1025  if not butler.datasetExists('fgcmFlaggedStars',
1026  fgcmcycle=self.config.cycleNumber-1):
1027  raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1028  (self.config.cycleNumber-1))
1029 
1030  # And additional dataset if we want reference calibration
1031  if self.config.doReferenceCalibration:
1032  if not butler.datasetExists('fgcmReferenceStars'):
1033  raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1034  "doReferenceCalibration is True.")
1035 
1036  def _loadParameters(self, butler):
1037  """
1038  Load FGCM parameters from a previous fit cycle
1039 
1040  Parameters
1041  ----------
1042  butler: `lsst.daf.persistence.Butler`
1043 
1044  Returns
1045  -------
1046  inParInfo: `numpy.ndarray`
1047  Numpy array parameter information formatted for input to fgcm
1048  inParameters: `numpy.ndarray`
1049  Numpy array parameter values formatted for input to fgcm
1050  inSuperStar: `numpy.array`
1051  Superstar flat formatted for input to fgcm
1052  """
1053 
1054  # note that we already checked that this is available
1055  parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1056 
1057  parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1058  parFitBands = np.array(parCat[0]['fitBands'].split(','))
1059  parNotFitBands = np.array(parCat[0]['notFitBands'].split(','))
1060 
1061  inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1062  ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1063  (parLutFilterNames.size, )),
1064  ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1065  ('NOTFITBANDS', parNotFitBands.dtype.str, (parNotFitBands.size, )),
1066  ('LNTAUUNIT', 'f8'),
1067  ('LNTAUSLOPEUNIT', 'f8'),
1068  ('ALPHAUNIT', 'f8'),
1069  ('LNPWVUNIT', 'f8'),
1070  ('LNPWVSLOPEUNIT', 'f8'),
1071  ('LNPWVQUADRATICUNIT', 'f8'),
1072  ('LNPWVGLOBALUNIT', 'f8'),
1073  ('O3UNIT', 'f8'),
1074  ('QESYSUNIT', 'f8'),
1075  ('FILTEROFFSETUNIT', 'f8'),
1076  ('HASEXTERNALPWV', 'i2'),
1077  ('HASEXTERNALTAU', 'i2')])
1078  inParInfo['NCCD'] = parCat['nCcd']
1079  inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1080  inParInfo['FITBANDS'][:] = parFitBands
1081  inParInfo['NOTFITBANDS'][:] = parNotFitBands
1082  inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1083  inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1084 
1085  inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1086  ('PARO3', 'f8', (parCat['parO3'].size, )),
1087  ('PARLNTAUINTERCEPT', 'f8',
1088  (parCat['parLnTauIntercept'].size, )),
1089  ('PARLNTAUSLOPE', 'f8',
1090  (parCat['parLnTauSlope'].size, )),
1091  ('PARLNPWVINTERCEPT', 'f8',
1092  (parCat['parLnPwvIntercept'].size, )),
1093  ('PARLNPWVSLOPE', 'f8',
1094  (parCat['parLnPwvSlope'].size, )),
1095  ('PARLNPWVQUADRATIC', 'f8',
1096  (parCat['parLnPwvQuadratic'].size, )),
1097  ('PARQESYSINTERCEPT', 'f8',
1098  (parCat['parQeSysIntercept'].size, )),
1099  ('COMPQESYSSLOPE', 'f8',
1100  (parCat['compQeSysSlope'].size, )),
1101  ('PARFILTEROFFSET', 'f8',
1102  (parCat['parFilterOffset'].size, )),
1103  ('PARFILTEROFFSETFITFLAG', 'i2',
1104  (parCat['parFilterOffsetFitFlag'].size, )),
1105  ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1106  ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1107  ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1108  (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1109  ('COMPABSTHROUGHPUT', 'f8',
1110  (parCat['compAbsThroughput'].size, )),
1111  ('COMPREFOFFSET', 'f8',
1112  (parCat['compRefOffset'].size, )),
1113  ('COMPREFSIGMA', 'f8',
1114  (parCat['compRefSigma'].size, )),
1115  ('COMPMIRRORCHROMATICITY', 'f8',
1116  (parCat['compMirrorChromaticity'].size, )),
1117  ('MIRRORCHROMATICITYPIVOT', 'f8',
1118  (parCat['mirrorChromaticityPivot'].size, )),
1119  ('COMPAPERCORRPIVOT', 'f8',
1120  (parCat['compAperCorrPivot'].size, )),
1121  ('COMPAPERCORRSLOPE', 'f8',
1122  (parCat['compAperCorrSlope'].size, )),
1123  ('COMPAPERCORRSLOPEERR', 'f8',
1124  (parCat['compAperCorrSlopeErr'].size, )),
1125  ('COMPAPERCORRRANGE', 'f8',
1126  (parCat['compAperCorrRange'].size, )),
1127  ('COMPMODELERREXPTIMEPIVOT', 'f8',
1128  (parCat['compModelErrExptimePivot'].size, )),
1129  ('COMPMODELERRFWHMPIVOT', 'f8',
1130  (parCat['compModelErrFwhmPivot'].size, )),
1131  ('COMPMODELERRSKYPIVOT', 'f8',
1132  (parCat['compModelErrSkyPivot'].size, )),
1133  ('COMPMODELERRPARS', 'f8',
1134  (parCat['compModelErrPars'].size, )),
1135  ('COMPEXPGRAY', 'f8',
1136  (parCat['compExpGray'].size, )),
1137  ('COMPVARGRAY', 'f8',
1138  (parCat['compVarGray'].size, )),
1139  ('COMPNGOODSTARPEREXP', 'i4',
1140  (parCat['compNGoodStarPerExp'].size, )),
1141  ('COMPSIGFGCM', 'f8',
1142  (parCat['compSigFgcm'].size, )),
1143  ('COMPSIGMACAL', 'f8',
1144  (parCat['compSigmaCal'].size, )),
1145  ('COMPRETRIEVEDLNPWV', 'f8',
1146  (parCat['compRetrievedLnPwv'].size, )),
1147  ('COMPRETRIEVEDLNPWVRAW', 'f8',
1148  (parCat['compRetrievedLnPwvRaw'].size, )),
1149  ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1150  (parCat['compRetrievedLnPwvFlag'].size, )),
1151  ('COMPRETRIEVEDTAUNIGHT', 'f8',
1152  (parCat['compRetrievedTauNight'].size, ))])
1153 
1154  inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1155  inParams['PARO3'][:] = parCat['parO3'][0, :]
1156  inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1157  inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1158  inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1159  inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1160  inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1161  inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1162  inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1163  inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1164  inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1165  inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1166  inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1167  inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1168  inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1169  inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1170  inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1171  inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1172  inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1173  inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1174  inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1175  inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1176  inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1177  inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1178  inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1179  inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1180  inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1181  inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1182  inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1183  inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1184  inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1185  inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1186  inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1187  inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1188  inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1189  inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1190 
1191  inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1192  inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1193 
1194  return (inParInfo, inParams, inSuperStar)
1195 
1196  def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1197  """
1198  Persist FGCM datasets through the butler.
1199 
1200  Parameters
1201  ----------
1202  butler: `lsst.daf.persistence.Butler`
1203  fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1204  Fgcm Fit cycle object
1205  """
1206 
1207  # Save the parameters
1208  parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1209 
1210  parSchema = afwTable.Schema()
1211 
1212  comma = ','
1213  lutFilterNameString = comma.join([n.decode('utf-8')
1214  for n in parInfo['LUTFILTERNAMES'][0]])
1215  fitBandString = comma.join([n.decode('utf-8')
1216  for n in parInfo['FITBANDS'][0]])
1217  notFitBandString = comma.join([n.decode('utf-8')
1218  for n in parInfo['NOTFITBANDS'][0]])
1219 
1220  parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1221  lutFilterNameString, fitBandString, notFitBandString)
1222  parCat = self._makeParCatalog(parSchema, parInfo, pars,
1223  fgcmFitCycle.fgcmPars.parSuperStarFlat,
1224  lutFilterNameString, fitBandString, notFitBandString)
1225 
1226  butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1227 
1228  # Save the indices of the flagged stars
1229  # (stars that have been (a) reserved from the fit for testing and
1230  # (b) bad stars that have failed quality checks.)
1231  flagStarSchema = self._makeFlagStarSchema()
1232  flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1233  flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1234 
1235  butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1236 
1237  # Save the zeropoint information and atmospheres only if desired
1238  if self.outputZeropoints:
1239  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1240  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1241 
1242  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1243  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1244 
1245  butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1246 
1247  # Save atmosphere values
1248  # These are generated by the same code that generates zeropoints
1249  atmSchema = makeAtmSchema()
1250  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1251 
1252  butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1253 
1254  # Save the standard stars (if configured)
1255  if self.outputStandards:
1256  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1257  stdSchema = makeStdSchema(len(goodBands))
1258  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1259 
1260  butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1261 
1262  def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1263  lutFilterNameString, fitBandString, notFitBandString):
1264  """
1265  Make the parameter persistence schema
1266 
1267  Parameters
1268  ----------
1269  parInfo: `numpy.ndarray`
1270  Parameter information returned by fgcm
1271  pars: `numpy.ndarray`
1272  Parameter values returned by fgcm
1273  parSuperStarFlat: `numpy.array`
1274  Superstar flat values returned by fgcm
1275  lutFilterNameString: `str`
1276  Combined string of all the lutFilterNames
1277  fitBandString: `str`
1278  Combined string of all the fitBands
1279  notFitBandString: `str`
1280  Combined string of all the bands not used in the fit
1281 
1282  Returns
1283  -------
1284  parSchema: `afwTable.schema`
1285  """
1286 
1287  parSchema = afwTable.Schema()
1288 
1289  # parameter info section
1290  parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1291  parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1292  size=len(lutFilterNameString))
1293  parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1294  size=len(fitBandString))
1295  parSchema.addField('notFitBands', type=str, doc='Bands that were not fit',
1296  size=len(notFitBandString))
1297  parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1298  parSchema.addField('lnTauSlopeUnit', type=np.float64,
1299  doc='Step units for ln(AOD) slope')
1300  parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1301  parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1302  parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1303  doc='Step units for ln(pwv) slope')
1304  parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1305  doc='Step units for ln(pwv) quadratic term')
1306  parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1307  doc='Step units for global ln(pwv) parameters')
1308  parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1309  parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1310  parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1311  parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1312  parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1313 
1314  # parameter section
1315  parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1316  size=pars['PARALPHA'].size)
1317  parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1318  size=pars['PARO3'].size)
1319  parSchema.addField('parLnTauIntercept', type='ArrayD',
1320  doc='ln(Tau) intercept parameter vector',
1321  size=pars['PARLNTAUINTERCEPT'].size)
1322  parSchema.addField('parLnTauSlope', type='ArrayD',
1323  doc='ln(Tau) slope parameter vector',
1324  size=pars['PARLNTAUSLOPE'].size)
1325  parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1326  size=pars['PARLNPWVINTERCEPT'].size)
1327  parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1328  size=pars['PARLNPWVSLOPE'].size)
1329  parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1330  size=pars['PARLNPWVQUADRATIC'].size)
1331  parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1332  size=pars['PARQESYSINTERCEPT'].size)
1333  parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1334  size=pars[0]['COMPQESYSSLOPE'].size)
1335  parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1336  size=pars['PARFILTEROFFSET'].size)
1337  parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1338  size=pars['PARFILTEROFFSETFITFLAG'].size)
1339  parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1340  doc='Global scale for retrieved ln(pwv)')
1341  parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1342  doc='Global offset for retrieved ln(pwv)')
1343  parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1344  doc='Nightly offset for retrieved ln(pwv)',
1345  size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1346  parSchema.addField('compAbsThroughput', type='ArrayD',
1347  doc='Absolute throughput (relative to transmission curves)',
1348  size=pars['COMPABSTHROUGHPUT'].size)
1349  parSchema.addField('compRefOffset', type='ArrayD',
1350  doc='Offset between reference stars and calibrated stars',
1351  size=pars['COMPREFOFFSET'].size)
1352  parSchema.addField('compRefSigma', type='ArrayD',
1353  doc='Width of reference star/calibrated star distribution',
1354  size=pars['COMPREFSIGMA'].size)
1355  parSchema.addField('compMirrorChromaticity', type='ArrayD',
1356  doc='Computed mirror chromaticity terms',
1357  size=pars['COMPMIRRORCHROMATICITY'].size)
1358  parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1359  doc='Mirror chromaticity pivot mjd',
1360  size=pars['MIRRORCHROMATICITYPIVOT'].size)
1361  parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1362  size=pars['COMPAPERCORRPIVOT'].size)
1363  parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1364  size=pars['COMPAPERCORRSLOPE'].size)
1365  parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1366  size=pars['COMPAPERCORRSLOPEERR'].size)
1367  parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1368  size=pars['COMPAPERCORRRANGE'].size)
1369  parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1370  size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1371  parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1372  size=pars['COMPMODELERRFWHMPIVOT'].size)
1373  parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1374  size=pars['COMPMODELERRSKYPIVOT'].size)
1375  parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1376  size=pars['COMPMODELERRPARS'].size)
1377  parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1378  size=pars['COMPEXPGRAY'].size)
1379  parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1380  size=pars['COMPVARGRAY'].size)
1381  parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1382  doc='Computed number of good stars per exposure',
1383  size=pars['COMPNGOODSTARPEREXP'].size)
1384  parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1385  size=pars['COMPSIGFGCM'].size)
1386  parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1387  size=pars['COMPSIGMACAL'].size)
1388  parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1389  size=pars['COMPRETRIEVEDLNPWV'].size)
1390  parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1391  size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1392  parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1393  size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1394  parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1395  size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1396  # superstarflat section
1397  parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1398  size=4)
1399  parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1400  size=parSuperStarFlat.size)
1401 
1402  return parSchema
1403 
1404  def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1405  lutFilterNameString, fitBandString, notFitBandString):
1406  """
1407  Make the FGCM parameter catalog for persistence
1408 
1409  Parameters
1410  ----------
1411  parSchema: `lsst.afw.table.Schema`
1412  Parameter catalog schema
1413  pars: `numpy.ndarray`
1414  FGCM parameters to put into parCat
1415  parSuperStarFlat: `numpy.array`
1416  FGCM superstar flat array to put into parCat
1417  lutFilterNameString: `str`
1418  Combined string of all the lutFilterNames
1419  fitBandString: `str`
1420  Combined string of all the fitBands
1421  notFitBandString: `str`
1422  Combined string of all the bands not used in the fit
1423 
1424  Returns
1425  -------
1426  parCat: `afwTable.BasicCatalog`
1427  Atmosphere and instrumental model parameter catalog for persistence
1428  """
1429 
1430  parCat = afwTable.BaseCatalog(parSchema)
1431  parCat.reserve(1)
1432 
1433  # The parameter catalog just has one row, with many columns for all the
1434  # atmosphere and instrument fit parameters
1435  rec = parCat.addNew()
1436 
1437  # info section
1438  rec['nCcd'] = parInfo['NCCD']
1439  rec['lutFilterNames'] = lutFilterNameString
1440  rec['fitBands'] = fitBandString
1441  rec['notFitBands'] = notFitBandString
1442  # note these are not currently supported here.
1443  rec['hasExternalPwv'] = 0
1444  rec['hasExternalTau'] = 0
1445 
1446  # parameter section
1447 
1448  scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1449 
1450  arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1451  'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1452  'parQeSysIntercept', 'compQeSysSlope',
1453  'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1454  'parFilterOffset', 'parFilterOffsetFitFlag',
1455  'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1456  'compMirrorChromaticity', 'mirrorChromaticityPivot',
1457  'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1458  'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1459  'compModelErrSkyPivot', 'compModelErrPars',
1460  'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1461  'compSigmaCal',
1462  'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1463  'compRetrievedTauNight']
1464 
1465  for scalarName in scalarNames:
1466  rec[scalarName] = pars[scalarName.upper()]
1467 
1468  for arrName in arrNames:
1469  rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1470 
1471  # superstar section
1472  rec['superstarSize'][:] = parSuperStarFlat.shape
1473  rec['superstar'][:] = parSuperStarFlat.flatten()
1474 
1475  return parCat
1476 
1477  def _makeFlagStarSchema(self):
1478  """
1479  Make the flagged-stars schema
1480 
1481  Returns
1482  -------
1483  flagStarSchema: `lsst.afw.table.Schema`
1484  """
1485 
1486  flagStarSchema = afwTable.Schema()
1487 
1488  flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1489  flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1490 
1491  return flagStarSchema
1492 
1493  def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1494  """
1495  Make the flagged star catalog for persistence
1496 
1497  Parameters
1498  ----------
1499  flagStarSchema: `lsst.afw.table.Schema`
1500  Flagged star schema
1501  flagStarStruct: `numpy.ndarray`
1502  Flagged star structure from fgcm
1503 
1504  Returns
1505  -------
1506  flagStarCat: `lsst.afw.table.BaseCatalog`
1507  Flagged star catalog for persistence
1508  """
1509 
1510  flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1511  flagStarCat.reserve(flagStarStruct.size)
1512  for i in range(flagStarStruct.size):
1513  flagStarCat.addNew()
1514 
1515  flagStarCat['objId'][:] = flagStarStruct['OBJID']
1516  flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1517 
1518  return flagStarCat
def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, lutFilterNameString, fitBandString, notFitBandString)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:44
def _makeFlagStarCat(self, flagStarSchema, flagStarStruct)
def translateFgcmLut(lutCat, filterMap)
Definition: utilities.py:196
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:378
def _persistFgcmDatasets(self, butler, fgcmFitCycle)
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:742
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:817
def makeStdSchema(nBands)
Definition: utilities.py:710
def translateVisitCatalog(visitCat)
Definition: utilities.py:327
def writeConfig(self, butler, clobber=False, doBackup=True)
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:675
def _makeParSchema(self, parInfo, pars, parSuperStarFlat, lutFilterNameString, fitBandString, notFitBandString)
def __init__(self, butler=None, kwargs)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:519
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:598