lsst.fgcmcal  21.0.0-10-g8d1d15d+4a5e44cf51
fgcmFitCycle.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Perform a single fit cycle of FGCM.
24 
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
28 for the global fit).
29 
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
34 """
35 
36 import sys
37 import traceback
38 import copy
39 
40 import numpy as np
41 
42 import lsst.pex.config as pexConfig
43 import lsst.pipe.base as pipeBase
44 from lsst.pipe.base import connectionTypes
45 import lsst.afw.table as afwTable
46 
47 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
48 from .utilities import extractReferenceMags
49 from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
50 from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51 from .sedterms import SedboundarytermDict, SedtermDict
52 from .utilities import lookupStaticCalibrations
53 
54 import fgcm
55 
56 __all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57 
58 
59 class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
60  dimensions=("instrument",),
61  defaultTemplates={"previousCycleNumber": "-1",
62  "cycleNumber": "0"}):
63  camera = connectionTypes.PrerequisiteInput(
64  doc="Camera instrument",
65  name="camera",
66  storageClass="Camera",
67  dimensions=("instrument",),
68  lookupFunction=lookupStaticCalibrations,
69  isCalibration=True,
70  )
71 
72  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
73  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
74  "chromatic corrections."),
75  name="fgcmLookUpTable",
76  storageClass="Catalog",
77  dimensions=("instrument",),
78  deferLoad=True,
79  )
80 
81  fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
82  doc="Catalog of visit information for fgcm",
83  name="fgcmVisitCatalog",
84  storageClass="Catalog",
85  dimensions=("instrument",),
86  deferLoad=True,
87  )
88 
89  fgcmStarObservations = connectionTypes.PrerequisiteInput(
90  doc="Catalog of star observations for fgcm",
91  name="fgcmStarObservations",
92  storageClass="Catalog",
93  dimensions=("instrument",),
94  deferLoad=True,
95  )
96 
97  fgcmStarIds = connectionTypes.PrerequisiteInput(
98  doc="Catalog of fgcm calibration star IDs",
99  name="fgcmStarIds",
100  storageClass="Catalog",
101  dimensions=("instrument",),
102  deferLoad=True,
103  )
104 
105  fgcmStarIndices = connectionTypes.PrerequisiteInput(
106  doc="Catalog of fgcm calibration star indices",
107  name="fgcmStarIndices",
108  storageClass="Catalog",
109  dimensions=("instrument",),
110  deferLoad=True,
111  )
112 
113  fgcmReferenceStars = connectionTypes.PrerequisiteInput(
114  doc="Catalog of fgcm-matched reference stars",
115  name="fgcmReferenceStars",
116  storageClass="Catalog",
117  dimensions=("instrument",),
118  deferLoad=True,
119  )
120 
121  fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
122  doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
123  name="fgcmFlaggedStars{previousCycleNumber}",
124  storageClass="Catalog",
125  dimensions=("instrument",),
126  deferLoad=True,
127  )
128 
129  fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
130  doc="Catalog of fgcm fit parameters from previous fit cycle",
131  name="fgcmFitParameters{previousCycleNumber}",
132  storageClass="Catalog",
133  dimensions=("instrument",),
134  deferLoad=True,
135  )
136 
137  fgcmFitParameters = connectionTypes.Output(
138  doc="Catalog of fgcm fit parameters from current fit cycle",
139  name="fgcmFitParameters{cycleNumber}",
140  storageClass="Catalog",
141  dimensions=("instrument",),
142  )
143 
144  fgcmFlaggedStars = connectionTypes.Output(
145  doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
146  name="fgcmFlaggedStars{cycleNumber}",
147  storageClass="Catalog",
148  dimensions=("instrument",),
149  )
150 
151  fgcmZeropoints = connectionTypes.Output(
152  doc="Catalog of fgcm zeropoint data from current fit cycle",
153  name="fgcmZeropoints{cycleNumber}",
154  storageClass="Catalog",
155  dimensions=("instrument",),
156  )
157 
158  fgcmAtmosphereParameters = connectionTypes.Output(
159  doc="Catalog of atmospheric fit parameters from current fit cycle",
160  name="fgcmAtmosphereParameters{cycleNumber}",
161  storageClass="Catalog",
162  dimensions=("instrument",),
163  )
164 
165  fgcmStandardStars = connectionTypes.Output(
166  doc="Catalog of standard star magnitudes from current fit cycle",
167  name="fgcmStandardStars{cycleNumber}",
168  storageClass="SimpleCatalog",
169  dimensions=("instrument",),
170  )
171 
172  def __init__(self, *, config=None):
173  super().__init__(config=config)
174 
175  if not config.doReferenceCalibration:
176  self.prerequisiteInputs.remove("fgcmReferenceStars")
177 
178  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
179  raise ValueError("cycleNumber must be of integer format")
180  if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
181  raise ValueError("previousCycleNumber must be of integer format")
182  if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
183  raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
184 
185  if int(config.connections.cycleNumber) == 0:
186  self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
187  self.prerequisiteInputs.remove("fgcmFitParametersInput")
188 
189  if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
190  self.outputs.remove("fgcmStandardStars")
191 
192  if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
193  self.outputs.remove("fgcmZeropoints")
194  self.outputs.remove("fgcmAtmosphereParameters")
195 
196 
197 class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
198  pipelineConnections=FgcmFitCycleConnections):
199  """Config for FgcmFitCycle"""
200 
201  bands = pexConfig.ListField(
202  doc="Bands to run calibration",
203  dtype=str,
204  default=[],
205  )
206  fitFlag = pexConfig.ListField(
207  doc=("Flag for which bands are directly constrained in the FGCM fit. "
208  "Bands set to 0 will have the atmosphere constrained from observations "
209  "in other bands on the same night. Must be same length as config.bands, "
210  "and matched band-by-band."),
211  dtype=int,
212  default=(0,),
213  optional=True,
214  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
215  "It will be removed after v20. Use fitBands instead."),
216  )
217  fitBands = pexConfig.ListField(
218  doc=("Bands to use in atmospheric fit. The bands not listed here will have "
219  "the atmosphere constrained from the 'fitBands' on the same night. "
220  "Must be a subset of `config.bands`"),
221  dtype=str,
222  default=[],
223  )
224  requiredFlag = pexConfig.ListField(
225  doc=("Flag for which bands are required for a star to be considered a calibration "
226  "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
227  "be same length as config.bands, and matched band-by-band."),
228  dtype=int,
229  default=(0,),
230  optional=True,
231  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
232  "It will be removed after v20. Use requiredBands instead."),
233  )
234  requiredBands = pexConfig.ListField(
235  doc=("Bands that are required for a star to be considered a calibration star. "
236  "Must be a subset of `config.bands`"),
237  dtype=str,
238  default=[],
239  )
240  filterMap = pexConfig.DictField(
241  doc="Mapping from 'filterName' to band.",
242  keytype=str,
243  itemtype=str,
244  default={},
245  deprecated=("This field is no longer used, and has been deprecated by "
246  "DM-28088. It will be removed after v22. Use "
247  "physicalFilterMap instead.")
248  )
249  # The following config will not be necessary after Gen2 retirement.
250  # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
251  # is easiest to access in the config file.
252  physicalFilterMap = pexConfig.DictField(
253  doc="Mapping from 'physicalFilter' to band.",
254  keytype=str,
255  itemtype=str,
256  default={},
257  )
258  doReferenceCalibration = pexConfig.Field(
259  doc="Use reference catalog as additional constraint on calibration",
260  dtype=bool,
261  default=True,
262  )
263  refStarSnMin = pexConfig.Field(
264  doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
265  dtype=float,
266  default=50.0,
267  )
268  refStarOutlierNSig = pexConfig.Field(
269  doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
270  "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
271  dtype=float,
272  default=4.0,
273  )
274  applyRefStarColorCuts = pexConfig.Field(
275  doc="Apply color cuts to reference stars?",
276  dtype=bool,
277  default=True,
278  )
279  nCore = pexConfig.Field(
280  doc="Number of cores to use",
281  dtype=int,
282  default=4,
283  )
284  nStarPerRun = pexConfig.Field(
285  doc="Number of stars to run in each chunk",
286  dtype=int,
287  default=200000,
288  )
289  nExpPerRun = pexConfig.Field(
290  doc="Number of exposures to run in each chunk",
291  dtype=int,
292  default=1000,
293  )
294  reserveFraction = pexConfig.Field(
295  doc="Fraction of stars to reserve for testing",
296  dtype=float,
297  default=0.1,
298  )
299  freezeStdAtmosphere = pexConfig.Field(
300  doc="Freeze atmosphere parameters to standard (for testing)",
301  dtype=bool,
302  default=False,
303  )
304  precomputeSuperStarInitialCycle = pexConfig.Field(
305  doc="Precompute superstar flat for initial cycle",
306  dtype=bool,
307  default=False,
308  )
309  superStarSubCcd = pexConfig.Field(
310  doc="Compute superstar flat on sub-ccd scale",
311  dtype=bool,
312  default=True,
313  optional=True,
314  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
315  "It will be removed after v20. Use superStarSubCcdDict instead."),
316  )
317  superStarSubCcdDict = pexConfig.DictField(
318  doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
319  "Must have one entry per band."),
320  keytype=str,
321  itemtype=bool,
322  default={},
323  )
324  superStarSubCcdChebyshevOrder = pexConfig.Field(
325  doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
326  "Global default is first-order polynomials, and should be overridden "
327  "on a camera-by-camera basis depending on the ISR."),
328  dtype=int,
329  default=1,
330  )
331  superStarSubCcdTriangular = pexConfig.Field(
332  doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
333  "suppress high-order cross terms?"),
334  dtype=bool,
335  default=False,
336  )
337  superStarSigmaClip = pexConfig.Field(
338  doc="Number of sigma to clip outliers when selecting for superstar flats",
339  dtype=float,
340  default=5.0,
341  )
342  focalPlaneSigmaClip = pexConfig.Field(
343  doc="Number of sigma to clip outliers per focal-plane.",
344  dtype=float,
345  default=4.0,
346  )
347  ccdGraySubCcd = pexConfig.Field(
348  doc="Compute CCD gray terms on sub-ccd scale",
349  dtype=bool,
350  default=False,
351  optional=True,
352  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
353  "It will be removed after v20. Use ccdGraySubCcdDict instead."),
354  )
355  ccdGraySubCcdDict = pexConfig.DictField(
356  doc=("Per-band specification on whether to compute achromatic per-ccd residual "
357  "('ccd gray') on a sub-ccd scale."),
358  keytype=str,
359  itemtype=bool,
360  default={},
361  )
362  ccdGraySubCcdChebyshevOrder = pexConfig.Field(
363  doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
364  dtype=int,
365  default=1,
366  )
367  ccdGraySubCcdTriangular = pexConfig.Field(
368  doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
369  "suppress high-order cross terms?"),
370  dtype=bool,
371  default=True,
372  )
373  ccdGrayFocalPlaneDict = pexConfig.DictField(
374  doc=("Per-band specification on whether to compute focal-plane residual "
375  "('ccd gray') corrections."),
376  keytype=str,
377  itemtype=bool,
378  default={},
379  )
380  ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
381  doc=("Minimum number of 'good' CCDs required to perform focal-plane "
382  "gray corrections. If there are fewer good CCDs then the gray "
383  "correction is computed per-ccd."),
384  dtype=int,
385  default=1,
386  )
387  ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
388  doc="Order of the 2D chebyshev polynomials for focal plane fit.",
389  dtype=int,
390  default=3,
391  )
392  cycleNumber = pexConfig.Field(
393  doc=("FGCM fit cycle number. This is automatically incremented after each run "
394  "and stage of outlier rejection. See cookbook for details."),
395  dtype=int,
396  default=None,
397  )
398  isFinalCycle = pexConfig.Field(
399  doc=("Is this the final cycle of the fitting? Will automatically compute final "
400  "selection of stars and photometric exposures, and will output zeropoints "
401  "and standard stars for use in fgcmOutputProducts"),
402  dtype=bool,
403  default=False,
404  )
405  maxIterBeforeFinalCycle = pexConfig.Field(
406  doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
407  "will always be 0 in the final cycle for cleanup and final selection."),
408  dtype=int,
409  default=50,
410  )
411  deltaMagBkgOffsetPercentile = pexConfig.Field(
412  doc=("Percentile brightest stars on a visit/ccd to use to compute net "
413  "offset from local background subtraction."),
414  dtype=float,
415  default=0.25,
416  )
417  deltaMagBkgPerCcd = pexConfig.Field(
418  doc=("Compute net offset from local background subtraction per-ccd? "
419  "Otherwise, use computation per visit."),
420  dtype=bool,
421  default=False,
422  )
423  utBoundary = pexConfig.Field(
424  doc="Boundary (in UTC) from day-to-day",
425  dtype=float,
426  default=None,
427  )
428  washMjds = pexConfig.ListField(
429  doc="Mirror wash MJDs",
430  dtype=float,
431  default=(0.0,),
432  )
433  epochMjds = pexConfig.ListField(
434  doc="Epoch boundaries in MJD",
435  dtype=float,
436  default=(0.0,),
437  )
438  minObsPerBand = pexConfig.Field(
439  doc="Minimum good observations per band",
440  dtype=int,
441  default=2,
442  )
443  # TODO: When DM-16511 is done, it will be possible to get the
444  # telescope latitude directly from the camera.
445  latitude = pexConfig.Field(
446  doc="Observatory latitude",
447  dtype=float,
448  default=None,
449  )
450  brightObsGrayMax = pexConfig.Field(
451  doc="Maximum gray extinction to be considered bright observation",
452  dtype=float,
453  default=0.15,
454  )
455  minStarPerCcd = pexConfig.Field(
456  doc=("Minimum number of good stars per CCD to be used in calibration fit. "
457  "CCDs with fewer stars will have their calibration estimated from other "
458  "CCDs in the same visit, with zeropoint error increased accordingly."),
459  dtype=int,
460  default=5,
461  )
462  minCcdPerExp = pexConfig.Field(
463  doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
464  "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
465  dtype=int,
466  default=5,
467  )
468  maxCcdGrayErr = pexConfig.Field(
469  doc="Maximum error on CCD gray offset to be considered photometric",
470  dtype=float,
471  default=0.05,
472  )
473  minStarPerExp = pexConfig.Field(
474  doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
475  "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
476  dtype=int,
477  default=600,
478  )
479  minExpPerNight = pexConfig.Field(
480  doc="Minimum number of good exposures/visits to consider a partly photometric night",
481  dtype=int,
482  default=10,
483  )
484  expGrayInitialCut = pexConfig.Field(
485  doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
486  "observations."),
487  dtype=float,
488  default=-0.25,
489  )
490  expGrayPhotometricCut = pexConfig.ListField(
491  doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
492  "Must be same length as config.bands, and matched band-by-band."),
493  dtype=float,
494  default=(0.0,),
495  optional=True,
496  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
497  "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
498  )
499  expGrayPhotometricCutDict = pexConfig.DictField(
500  doc=("Per-band specification on maximum (negative) achromatic exposure residual "
501  "('gray term') for a visit to be considered photometric. Must have one "
502  "entry per band. Broad-band filters should be -0.05."),
503  keytype=str,
504  itemtype=float,
505  default={},
506  )
507  expGrayHighCut = pexConfig.ListField(
508  doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
509  "Must be same length as config.bands, and matched band-by-band."),
510  dtype=float,
511  default=(0.0,),
512  optional=True,
513  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
514  "It will be removed after v20. Use expGrayHighCutDict instead."),
515  )
516  expGrayHighCutDict = pexConfig.DictField(
517  doc=("Per-band specification on maximum (positive) achromatic exposure residual "
518  "('gray term') for a visit to be considered photometric. Must have one "
519  "entry per band. Broad-band filters should be 0.2."),
520  keytype=str,
521  itemtype=float,
522  default={},
523  )
524  expGrayRecoverCut = pexConfig.Field(
525  doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
526  "Visits with more gray extinction will only get CCD zeropoints if there are "
527  "sufficient star observations (minStarPerCcd) on that CCD."),
528  dtype=float,
529  default=-1.0,
530  )
531  expVarGrayPhotometricCut = pexConfig.Field(
532  doc="Maximum exposure variance to be considered possibly photometric",
533  dtype=float,
534  default=0.0005,
535  optional=True,
536  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
537  "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
538  )
539  expVarGrayPhotometricCutDict = pexConfig.DictField(
540  doc=("Per-band specification on maximum exposure variance to be considered possibly "
541  "photometric. Must have one entry per band. Broad-band filters should be "
542  "0.0005."),
543  keytype=str,
544  itemtype=float,
545  default={},
546  )
547  expGrayErrRecoverCut = pexConfig.Field(
548  doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
549  "Visits with more gray variance will only get CCD zeropoints if there are "
550  "sufficient star observations (minStarPerCcd) on that CCD."),
551  dtype=float,
552  default=0.05,
553  )
554  aperCorrFitNBins = pexConfig.Field(
555  doc=("Number of aperture bins used in aperture correction fit. When set to 0"
556  "no fit will be performed, and the config.aperCorrInputSlopes will be "
557  "used if available."),
558  dtype=int,
559  default=10,
560  )
561  aperCorrInputSlopes = pexConfig.ListField(
562  doc=("Aperture correction input slope parameters. These are used on the first "
563  "fit iteration, and aperture correction parameters will be updated from "
564  "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
565  "when there is insufficient data to fit the parameters (e.g. tract mode). "
566  "If set, must be same length as config.bands, and matched band-by-band."),
567  dtype=float,
568  default=[],
569  optional=True,
570  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
571  "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
572  )
573  aperCorrInputSlopeDict = pexConfig.DictField(
574  doc=("Per-band specification of aperture correction input slope parameters. These "
575  "are used on the first fit iteration, and aperture correction parameters will "
576  "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
577  "to set this when there is insufficient data to fit the parameters (e.g. "
578  "tract mode)."),
579  keytype=str,
580  itemtype=float,
581  default={},
582  )
583  sedFudgeFactors = pexConfig.ListField(
584  doc=("Fudge factors for computing linear SED from colors. Must be same length as "
585  "config.bands, and matched band-by-band."),
586  dtype=float,
587  default=(0,),
588  optional=True,
589  deprecated=("This field has been deprecated and will be removed after v20. "
590  "Please use sedSlopeTermMap and sedSlopeMap."),
591  )
592  sedboundaryterms = pexConfig.ConfigField(
593  doc="Mapping from bands to SED boundary term names used is sedterms.",
594  dtype=SedboundarytermDict,
595  )
596  sedterms = pexConfig.ConfigField(
597  doc="Mapping from terms to bands for fgcm linear SED approximations.",
598  dtype=SedtermDict,
599  )
600  sigFgcmMaxErr = pexConfig.Field(
601  doc="Maximum mag error for fitting sigma_FGCM",
602  dtype=float,
603  default=0.01,
604  )
605  sigFgcmMaxEGray = pexConfig.ListField(
606  doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
607  "May be 1 element (same for all bands) or the same length as config.bands."),
608  dtype=float,
609  default=(0.05,),
610  optional=True,
611  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
612  "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
613  )
614  sigFgcmMaxEGrayDict = pexConfig.DictField(
615  doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
616  "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
617  "should be 0.05."),
618  keytype=str,
619  itemtype=float,
620  default={},
621  )
622  ccdGrayMaxStarErr = pexConfig.Field(
623  doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
624  "computation"),
625  dtype=float,
626  default=0.10,
627  )
628  approxThroughput = pexConfig.ListField(
629  doc=("Approximate overall throughput at start of calibration observations. "
630  "May be 1 element (same for all bands) or the same length as config.bands."),
631  dtype=float,
632  default=(1.0, ),
633  optional=True,
634  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
635  "It will be removed after v20. Use approxThroughputDict instead."),
636  )
637  approxThroughputDict = pexConfig.DictField(
638  doc=("Per-band specification of the approximate overall throughput at the start of "
639  "calibration observations. Must have one entry per band. Typically should "
640  "be 1.0."),
641  keytype=str,
642  itemtype=float,
643  default={},
644  )
645  sigmaCalRange = pexConfig.ListField(
646  doc="Allowed range for systematic error floor estimation",
647  dtype=float,
648  default=(0.001, 0.003),
649  )
650  sigmaCalFitPercentile = pexConfig.ListField(
651  doc="Magnitude percentile range to fit systematic error floor",
652  dtype=float,
653  default=(0.05, 0.15),
654  )
655  sigmaCalPlotPercentile = pexConfig.ListField(
656  doc="Magnitude percentile range to plot systematic error floor",
657  dtype=float,
658  default=(0.05, 0.95),
659  )
660  sigma0Phot = pexConfig.Field(
661  doc="Systematic error floor for all zeropoints",
662  dtype=float,
663  default=0.003,
664  )
665  mapLongitudeRef = pexConfig.Field(
666  doc="Reference longitude for plotting maps",
667  dtype=float,
668  default=0.0,
669  )
670  mapNSide = pexConfig.Field(
671  doc="Healpix nside for plotting maps",
672  dtype=int,
673  default=256,
674  )
675  outfileBase = pexConfig.Field(
676  doc="Filename start for plot output files",
677  dtype=str,
678  default=None,
679  )
680  starColorCuts = pexConfig.ListField(
681  doc="Encoded star-color cuts (to be cleaned up)",
682  dtype=str,
683  default=("NO_DATA",),
684  )
685  colorSplitIndices = pexConfig.ListField(
686  doc="Band indices to use to split stars by color",
687  dtype=int,
688  default=None,
689  optional=True,
690  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
691  "It will be removed after v20. Use colorSplitBands instead."),
692  )
693  colorSplitBands = pexConfig.ListField(
694  doc="Band names to use to split stars by color. Must have 2 entries.",
695  dtype=str,
696  length=2,
697  default=('g', 'i'),
698  )
699  modelMagErrors = pexConfig.Field(
700  doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
701  dtype=bool,
702  default=True,
703  )
704  useQuadraticPwv = pexConfig.Field(
705  doc="Model PWV with a quadratic term for variation through the night?",
706  dtype=bool,
707  default=False,
708  )
709  instrumentParsPerBand = pexConfig.Field(
710  doc=("Model instrumental parameters per band? "
711  "Otherwise, instrumental parameters (QE changes with time) are "
712  "shared among all bands."),
713  dtype=bool,
714  default=False,
715  )
716  instrumentSlopeMinDeltaT = pexConfig.Field(
717  doc=("Minimum time change (in days) between observations to use in constraining "
718  "instrument slope."),
719  dtype=float,
720  default=20.0,
721  )
722  fitMirrorChromaticity = pexConfig.Field(
723  doc="Fit (intraband) mirror chromatic term?",
724  dtype=bool,
725  default=False,
726  )
727  coatingMjds = pexConfig.ListField(
728  doc="Mirror coating dates in MJD",
729  dtype=float,
730  default=(0.0,),
731  )
732  outputStandardsBeforeFinalCycle = pexConfig.Field(
733  doc="Output standard stars prior to final cycle? Used in debugging.",
734  dtype=bool,
735  default=False,
736  )
737  outputZeropointsBeforeFinalCycle = pexConfig.Field(
738  doc="Output standard stars prior to final cycle? Used in debugging.",
739  dtype=bool,
740  default=False,
741  )
742  useRepeatabilityForExpGrayCuts = pexConfig.ListField(
743  doc=("Use star repeatability (instead of exposures) for computing photometric "
744  "cuts? Recommended for tract mode or bands with few exposures. "
745  "May be 1 element (same for all bands) or the same length as config.bands."),
746  dtype=bool,
747  default=(False,),
748  optional=True,
749  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
750  "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
751  )
752  useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
753  doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
754  "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
755  keytype=str,
756  itemtype=bool,
757  default={},
758  )
759  autoPhotometricCutNSig = pexConfig.Field(
760  doc=("Number of sigma for automatic computation of (low) photometric cut. "
761  "Cut is based on exposure gray width (per band), unless "
762  "useRepeatabilityForExpGrayCuts is set, in which case the star "
763  "repeatability is used (also per band)."),
764  dtype=float,
765  default=3.0,
766  )
767  autoHighCutNSig = pexConfig.Field(
768  doc=("Number of sigma for automatic computation of (high) outlier cut. "
769  "Cut is based on exposure gray width (per band), unless "
770  "useRepeatabilityForExpGrayCuts is set, in which case the star "
771  "repeatability is used (also per band)."),
772  dtype=float,
773  default=4.0,
774  )
775  quietMode = pexConfig.Field(
776  doc="Be less verbose with logging.",
777  dtype=bool,
778  default=False,
779  )
780  randomSeed = pexConfig.Field(
781  doc="Random seed for fgcm for consistency in tests.",
782  dtype=int,
783  default=None,
784  optional=True,
785  )
786 
787  def validate(self):
788  super().validate()
789 
790  if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
791  msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
792  raise RuntimeError(msg)
793  if self.connections.cycleNumber != str(self.cycleNumber):
794  msg = "cycleNumber in template must be equal to connections.cycleNumber"
795  raise RuntimeError(msg)
796 
797  for band in self.fitBands:
798  if band not in self.bands:
799  msg = 'fitBand %s not in bands' % (band)
800  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
801  for band in self.requiredBands:
802  if band not in self.bands:
803  msg = 'requiredBand %s not in bands' % (band)
804  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
805  for band in self.colorSplitBands:
806  if band not in self.bands:
807  msg = 'colorSplitBand %s not in bands' % (band)
808  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
809  for band in self.bands:
810  if band not in self.superStarSubCcdDict:
811  msg = 'band %s not in superStarSubCcdDict' % (band)
812  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
813  self, msg)
814  if band not in self.ccdGraySubCcdDict:
815  msg = 'band %s not in ccdGraySubCcdDict' % (band)
816  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
817  self, msg)
818  if band not in self.expGrayPhotometricCutDict:
819  msg = 'band %s not in expGrayPhotometricCutDict' % (band)
820  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
821  self, msg)
822  if band not in self.expGrayHighCutDict:
823  msg = 'band %s not in expGrayHighCutDict' % (band)
824  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
825  self, msg)
826  if band not in self.expVarGrayPhotometricCutDict:
827  msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
828  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
829  self, msg)
830  if band not in self.sigFgcmMaxEGrayDict:
831  msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
832  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
833  self, msg)
834  if band not in self.approxThroughputDict:
835  msg = 'band %s not in approxThroughputDict' % (band)
836  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
837  self, msg)
838  if band not in self.useRepeatabilityForExpGrayCutsDict:
839  msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
840  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
841  self, msg)
842 
843 
844 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
845  """Subclass of TaskRunner for fgcmFitCycleTask
846 
847  fgcmFitCycleTask.run() takes one argument, the butler, and uses
848  stars and visits previously extracted from dataRefs by
849  fgcmBuildStars.
850  This Runner does not perform any dataRef parallelization, but the FGCM
851  code called by the Task uses python multiprocessing (see the "ncores"
852  config option).
853  """
854 
855  @staticmethod
856  def getTargetList(parsedCmd):
857  """
858  Return a list with one element, the butler.
859  """
860  return [parsedCmd.butler]
861 
862  def __call__(self, butler):
863  """
864  Parameters
865  ----------
866  butler: `lsst.daf.persistence.Butler`
867 
868  Returns
869  -------
870  exitStatus: `list` with `pipeBase.Struct`
871  exitStatus (0: success; 1: failure)
872  """
873 
874  task = self.TaskClass(config=self.config, log=self.log)
875 
876  exitStatus = 0
877  if self.doRaise:
878  task.runDataRef(butler)
879  else:
880  try:
881  task.runDataRef(butler)
882  except Exception as e:
883  exitStatus = 1
884  task.log.fatal("Failed: %s" % e)
885  if not isinstance(e, pipeBase.TaskError):
886  traceback.print_exc(file=sys.stderr)
887 
888  task.writeMetadata(butler)
889 
890  # The task does not return any results:
891  return [pipeBase.Struct(exitStatus=exitStatus)]
892 
893  def run(self, parsedCmd):
894  """
895  Run the task, with no multiprocessing
896 
897  Parameters
898  ----------
899  parsedCmd: ArgumentParser parsed command line
900  """
901 
902  resultList = []
903 
904  if self.precall(parsedCmd):
905  targetList = self.getTargetList(parsedCmd)
906  # make sure that we only get 1
907  resultList = self(targetList[0])
908 
909  return resultList
910 
911 
912 class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
913  """
914  Run Single fit cycle for FGCM global calibration
915  """
916 
917  ConfigClass = FgcmFitCycleConfig
918  RunnerClass = FgcmFitCycleRunner
919  _DefaultName = "fgcmFitCycle"
920 
921  def __init__(self, butler=None, initInputs=None, **kwargs):
922  super().__init__(**kwargs)
923 
924  # no saving of metadata for now
925  def _getMetadataName(self):
926  return None
927 
928  def runQuantum(self, butlerQC, inputRefs, outputRefs):
929  camera = butlerQC.get(inputRefs.camera)
930 
931  dataRefDict = {}
932 
933  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
934  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
935  dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
936  dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
937  dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
938  if self.config.doReferenceCalibration:
939  dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
940  if self.config.cycleNumber > 0:
941  dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
942  dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
943 
944  fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
945 
946  butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
947  butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
948  if self.outputZeropoints:
949  butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
950  butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
951  if self.outputStandards:
952  butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
953 
954  @pipeBase.timeMethod
955  def runDataRef(self, butler):
956  """
957  Run a single fit cycle for FGCM
958 
959  Parameters
960  ----------
961  butler: `lsst.daf.persistence.Butler`
962  """
963  self._checkDatasetsExist(butler)
964 
965  dataRefDict = {}
966  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
967  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
968  dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
969  dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
970  dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
971  if self.config.doReferenceCalibration:
972  dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
973  if self.config.cycleNumber > 0:
974  lastCycle = self.config.cycleNumber - 1
975  dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
976  fgcmcycle=lastCycle)
977  dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
978  fgcmcycle=lastCycle)
979 
980  camera = butler.get('camera')
981  fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
982 
983  butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
984  fgcmcycle=self.config.cycleNumber)
985  butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
986  fgcmcycle=self.config.cycleNumber)
987  if self.outputZeropoints:
988  butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
989  fgcmcycle=self.config.cycleNumber)
990  butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
991  fgcmcycle=self.config.cycleNumber)
992  if self.outputStandards:
993  butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
994  fgcmcycle=self.config.cycleNumber)
995 
996  def writeConfig(self, butler, clobber=False, doBackup=True):
997  """Write the configuration used for processing the data, or check that an existing
998  one is equal to the new one if present. This is an override of the regular
999  version from pipe_base that knows about fgcmcycle.
1000 
1001  Parameters
1002  ----------
1003  butler : `lsst.daf.persistence.Butler`
1004  Data butler used to write the config. The config is written to dataset type
1005  `CmdLineTask._getConfigName`.
1006  clobber : `bool`, optional
1007  A boolean flag that controls what happens if a config already has been saved:
1008  - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1009  - `False`: raise `TaskError` if this config does not match the existing config.
1010  doBackup : `bool`, optional
1011  Set to `True` to backup the config files if clobbering.
1012  """
1013  configName = self._getConfigName()
1014  if configName is None:
1015  return
1016  if clobber:
1017  butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1018  elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1019  # this may be subject to a race condition; see #2789
1020  try:
1021  oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1022  except Exception as exc:
1023  raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1024  (configName, exc))
1025 
1026  def logConfigMismatch(msg):
1027  self.log.fatal("Comparing configuration: %s", msg)
1028 
1029  if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1030  raise pipeBase.TaskError(
1031  f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1032  " must be consistent within the same output repo (override with --clobber-config)")
1033  else:
1034  butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1035 
1036  def _fgcmFitCycle(self, camera, dataRefDict):
1037  """
1038  Run the fit cycle
1039 
1040  Parameters
1041  ----------
1042  camera : `lsst.afw.cameraGeom.Camera`
1043  dataRefDict : `dict`
1044  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1045  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1046  dataRef dictionary with keys:
1047 
1048  ``"fgcmLookUpTable"``
1049  dataRef for the FGCM look-up table.
1050  ``"fgcmVisitCatalog"``
1051  dataRef for visit summary catalog.
1052  ``"fgcmStarObservations"``
1053  dataRef for star observation catalog.
1054  ``"fgcmStarIds"``
1055  dataRef for star id catalog.
1056  ``"fgcmStarIndices"``
1057  dataRef for star index catalog.
1058  ``"fgcmReferenceStars"``
1059  dataRef for matched reference star catalog.
1060  ``"fgcmFlaggedStars"``
1061  dataRef for flagged star catalog.
1062  ``"fgcmFitParameters"``
1063  dataRef for fit parameter catalog.
1064 
1065  Returns
1066  -------
1067  fgcmDatasetDict : `dict`
1068  Dictionary of datasets to persist.
1069  """
1070  # Set defaults on whether to output standards and zeropoints
1071  self.maxIter = self.config.maxIterBeforeFinalCycle
1072  self.outputStandards = self.config.outputStandardsBeforeFinalCycle
1073  self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
1074  self.resetFitParameters = True
1075 
1076  if self.config.isFinalCycle:
1077  # This is the final fit cycle, so we do not want to reset fit
1078  # parameters, we want to run a final "clean-up" with 0 fit iterations,
1079  # and we always want to output standards and zeropoints
1080  self.maxIter = 0
1081  self.outputStandards = True
1082  self.outputZeropoints = True
1083  self.resetFitParameters = False
1084 
1085  configDict = makeConfigDict(self.config, self.log, camera,
1086  self.maxIter, self.resetFitParameters,
1087  self.outputZeropoints)
1088 
1089  lutCat = dataRefDict['fgcmLookUpTable'].get()
1090  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1091  dict(self.config.physicalFilterMap))
1092  del lutCat
1093 
1094  # next we need the exposure/visit information
1095  visitCat = dataRefDict['fgcmVisitCatalog'].get()
1096  fgcmExpInfo = translateVisitCatalog(visitCat)
1097  del visitCat
1098 
1099  # Use the first orientation.
1100  # TODO: DM-21215 will generalize to arbitrary camera orientations
1101  ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1102 
1103  noFitsDict = {'lutIndex': lutIndexVals,
1104  'lutStd': lutStd,
1105  'expInfo': fgcmExpInfo,
1106  'ccdOffsets': ccdOffsets}
1107 
1108  # set up the fitter object
1109  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1110  noFitsDict=noFitsDict, noOutput=True)
1111 
1112  # create the parameter object
1113  if (fgcmFitCycle.initialCycle):
1114  # cycle = 0, initial cycle
1115  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1116  fgcmLut,
1117  fgcmExpInfo)
1118  else:
1119  inParInfo, inParams, inSuperStar = self._loadParameters(dataRefDict['fgcmFitParameters'])
1120  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1121  fgcmExpInfo,
1122  inParInfo,
1123  inParams,
1124  inSuperStar)
1125 
1126  # set up the stars...
1127  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1128 
1129  starObs = dataRefDict['fgcmStarObservations'].get()
1130  starIds = dataRefDict['fgcmStarIds'].get()
1131  starIndices = dataRefDict['fgcmStarIndices'].get()
1132 
1133  # grab the flagged stars if available
1134  if 'fgcmFlaggedStars' in dataRefDict:
1135  flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1136  flagId = flaggedStars['objId'][:]
1137  flagFlag = flaggedStars['objFlag'][:]
1138  else:
1139  flaggedStars = None
1140  flagId = None
1141  flagFlag = None
1142 
1143  if self.config.doReferenceCalibration:
1144  refStars = dataRefDict['fgcmReferenceStars'].get()
1145 
1146  refMag, refMagErr = extractReferenceMags(refStars,
1147  self.config.bands,
1148  self.config.physicalFilterMap)
1149  refId = refStars['fgcm_id'][:]
1150  else:
1151  refStars = None
1152  refId = None
1153  refMag = None
1154  refMagErr = None
1155 
1156  # match star observations to visits
1157  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1158  # actually be transferred into fgcm using the indexing below.
1159  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1160 
1161  # The fgcmStars.loadStars method will copy all the star information into
1162  # special shared memory objects that will not blow up the memory usage when
1163  # used with python multiprocessing. Once all the numbers are copied,
1164  # it is necessary to release all references to the objects that previously
1165  # stored the data to ensure that the garbage collector can clear the memory,
1166  # and ensure that this memory is not copied when multiprocessing kicks in.
1167 
1168  # We determine the conversion from the native units (typically radians) to
1169  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1170  # numpy arrays rather than Angles, which would we approximately 600x slower.
1171  conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1172 
1173  fgcmStars.loadStars(fgcmPars,
1174  starObs['visit'][starIndices['obsIndex']],
1175  starObs['ccd'][starIndices['obsIndex']],
1176  starObs['ra'][starIndices['obsIndex']] * conv,
1177  starObs['dec'][starIndices['obsIndex']] * conv,
1178  starObs['instMag'][starIndices['obsIndex']],
1179  starObs['instMagErr'][starIndices['obsIndex']],
1180  fgcmExpInfo['FILTERNAME'][visitIndex],
1181  starIds['fgcm_id'][:],
1182  starIds['ra'][:],
1183  starIds['dec'][:],
1184  starIds['obsArrIndex'][:],
1185  starIds['nObs'][:],
1186  obsX=starObs['x'][starIndices['obsIndex']],
1187  obsY=starObs['y'][starIndices['obsIndex']],
1188  obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1189  psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1190  refID=refId,
1191  refMag=refMag,
1192  refMagErr=refMagErr,
1193  flagID=flagId,
1194  flagFlag=flagFlag,
1195  computeNobs=True)
1196 
1197  # Release all references to temporary objects holding star data (see above)
1198  del starObs
1199  del starIds
1200  del starIndices
1201  del flagId
1202  del flagFlag
1203  del flaggedStars
1204  del refStars
1205  del refId
1206  del refMag
1207  del refMagErr
1208 
1209  # and set the bits in the cycle object
1210  fgcmFitCycle.setLUT(fgcmLut)
1211  fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1212  fgcmFitCycle.setPars(fgcmPars)
1213 
1214  # finish the setup
1215  fgcmFitCycle.finishSetup()
1216 
1217  # and run
1218  fgcmFitCycle.run()
1219 
1220 
1223 
1224  fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1225 
1226  # Output the config for the next cycle
1227  # We need to make a copy since the input one has been frozen
1228 
1229  updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1230  i, b in enumerate(self.config.bands)}
1231  updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1232  i, band in enumerate(self.config.bands)}
1233 
1234  outConfig = copy.copy(self.config)
1235  outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
1236  precomputeSuperStarInitialCycle=False,
1237  freezeStdAtmosphere=False,
1238  expGrayPhotometricCutDict=updatedPhotometricCutDict,
1239  expGrayHighCutDict=updatedHighCutDict)
1240 
1241  outConfig.connections.update(previousCycleNumber=str(self.config.cycleNumber),
1242  cycleNumber=str(self.config.cycleNumber + 1))
1243 
1244  configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1245  outConfig.cycleNumber)
1246  outConfig.save(configFileName)
1247 
1248  if self.config.isFinalCycle == 1:
1249  # We are done, ready to output products
1250  self.log.info("Everything is in place to run fgcmOutputProducts.py")
1251  else:
1252  self.log.info("Saved config for next cycle to %s" % (configFileName))
1253  self.log.info("Be sure to look at:")
1254  self.log.info(" config.expGrayPhotometricCut")
1255  self.log.info(" config.expGrayHighCut")
1256  self.log.info("If you are satisfied with the fit, please set:")
1257  self.log.info(" config.isFinalCycle = True")
1258 
1259  return fgcmDatasetDict
1260 
1261  def _checkDatasetsExist(self, butler):
1262  """
1263  Check if necessary datasets exist to run fgcmFitCycle
1264 
1265  Parameters
1266  ----------
1267  butler: `lsst.daf.persistence.Butler`
1268 
1269  Raises
1270  ------
1271  RuntimeError
1272  If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1273  fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1274  If cycleNumber > 0, then also checks for fgcmFitParameters,
1275  fgcmFlaggedStars.
1276  """
1277 
1278  if not butler.datasetExists('fgcmVisitCatalog'):
1279  raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1280  if not butler.datasetExists('fgcmStarObservations'):
1281  raise RuntimeError("Could not find fgcmStarObservations in repo!")
1282  if not butler.datasetExists('fgcmStarIds'):
1283  raise RuntimeError("Could not find fgcmStarIds in repo!")
1284  if not butler.datasetExists('fgcmStarIndices'):
1285  raise RuntimeError("Could not find fgcmStarIndices in repo!")
1286  if not butler.datasetExists('fgcmLookUpTable'):
1287  raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1288 
1289  # Need additional datasets if we are not the initial cycle
1290  if (self.config.cycleNumber > 0):
1291  if not butler.datasetExists('fgcmFitParameters',
1292  fgcmcycle=self.config.cycleNumber-1):
1293  raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1294  (self.config.cycleNumber-1))
1295  if not butler.datasetExists('fgcmFlaggedStars',
1296  fgcmcycle=self.config.cycleNumber-1):
1297  raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1298  (self.config.cycleNumber-1))
1299 
1300  # And additional dataset if we want reference calibration
1301  if self.config.doReferenceCalibration:
1302  if not butler.datasetExists('fgcmReferenceStars'):
1303  raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1304  "doReferenceCalibration is True.")
1305 
1306  def _loadParameters(self, parDataRef):
1307  """
1308  Load FGCM parameters from a previous fit cycle
1309 
1310  Parameters
1311  ----------
1312  parDataRef : `lsst.daf.persistence.ButlerDataRef` or `lsst.daf.butler.DeferredDatasetHandle`
1313  dataRef for previous fit parameter catalog.
1314 
1315  Returns
1316  -------
1317  inParInfo: `numpy.ndarray`
1318  Numpy array parameter information formatted for input to fgcm
1319  inParameters: `numpy.ndarray`
1320  Numpy array parameter values formatted for input to fgcm
1321  inSuperStar: `numpy.array`
1322  Superstar flat formatted for input to fgcm
1323  """
1324  # note that we already checked that this is available
1325  parCat = parDataRef.get()
1326 
1327  parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1328  parFitBands = np.array(parCat[0]['fitBands'].split(','))
1329 
1330  inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1331  ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1332  (parLutFilterNames.size, )),
1333  ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1334  ('LNTAUUNIT', 'f8'),
1335  ('LNTAUSLOPEUNIT', 'f8'),
1336  ('ALPHAUNIT', 'f8'),
1337  ('LNPWVUNIT', 'f8'),
1338  ('LNPWVSLOPEUNIT', 'f8'),
1339  ('LNPWVQUADRATICUNIT', 'f8'),
1340  ('LNPWVGLOBALUNIT', 'f8'),
1341  ('O3UNIT', 'f8'),
1342  ('QESYSUNIT', 'f8'),
1343  ('FILTEROFFSETUNIT', 'f8'),
1344  ('HASEXTERNALPWV', 'i2'),
1345  ('HASEXTERNALTAU', 'i2')])
1346  inParInfo['NCCD'] = parCat['nCcd']
1347  inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1348  inParInfo['FITBANDS'][:] = parFitBands
1349  inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1350  inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1351 
1352  inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1353  ('PARO3', 'f8', (parCat['parO3'].size, )),
1354  ('PARLNTAUINTERCEPT', 'f8',
1355  (parCat['parLnTauIntercept'].size, )),
1356  ('PARLNTAUSLOPE', 'f8',
1357  (parCat['parLnTauSlope'].size, )),
1358  ('PARLNPWVINTERCEPT', 'f8',
1359  (parCat['parLnPwvIntercept'].size, )),
1360  ('PARLNPWVSLOPE', 'f8',
1361  (parCat['parLnPwvSlope'].size, )),
1362  ('PARLNPWVQUADRATIC', 'f8',
1363  (parCat['parLnPwvQuadratic'].size, )),
1364  ('PARQESYSINTERCEPT', 'f8',
1365  (parCat['parQeSysIntercept'].size, )),
1366  ('COMPQESYSSLOPE', 'f8',
1367  (parCat['compQeSysSlope'].size, )),
1368  ('PARFILTEROFFSET', 'f8',
1369  (parCat['parFilterOffset'].size, )),
1370  ('PARFILTEROFFSETFITFLAG', 'i2',
1371  (parCat['parFilterOffsetFitFlag'].size, )),
1372  ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1373  ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1374  ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1375  (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1376  ('COMPABSTHROUGHPUT', 'f8',
1377  (parCat['compAbsThroughput'].size, )),
1378  ('COMPREFOFFSET', 'f8',
1379  (parCat['compRefOffset'].size, )),
1380  ('COMPREFSIGMA', 'f8',
1381  (parCat['compRefSigma'].size, )),
1382  ('COMPMIRRORCHROMATICITY', 'f8',
1383  (parCat['compMirrorChromaticity'].size, )),
1384  ('MIRRORCHROMATICITYPIVOT', 'f8',
1385  (parCat['mirrorChromaticityPivot'].size, )),
1386  ('COMPMEDIANSEDSLOPE', 'f8',
1387  (parCat['compMedianSedSlope'].size, )),
1388  ('COMPAPERCORRPIVOT', 'f8',
1389  (parCat['compAperCorrPivot'].size, )),
1390  ('COMPAPERCORRSLOPE', 'f8',
1391  (parCat['compAperCorrSlope'].size, )),
1392  ('COMPAPERCORRSLOPEERR', 'f8',
1393  (parCat['compAperCorrSlopeErr'].size, )),
1394  ('COMPAPERCORRRANGE', 'f8',
1395  (parCat['compAperCorrRange'].size, )),
1396  ('COMPMODELERREXPTIMEPIVOT', 'f8',
1397  (parCat['compModelErrExptimePivot'].size, )),
1398  ('COMPMODELERRFWHMPIVOT', 'f8',
1399  (parCat['compModelErrFwhmPivot'].size, )),
1400  ('COMPMODELERRSKYPIVOT', 'f8',
1401  (parCat['compModelErrSkyPivot'].size, )),
1402  ('COMPMODELERRPARS', 'f8',
1403  (parCat['compModelErrPars'].size, )),
1404  ('COMPEXPGRAY', 'f8',
1405  (parCat['compExpGray'].size, )),
1406  ('COMPVARGRAY', 'f8',
1407  (parCat['compVarGray'].size, )),
1408  ('COMPEXPDELTAMAGBKG', 'f8',
1409  (parCat['compExpDeltaMagBkg'].size, )),
1410  ('COMPNGOODSTARPEREXP', 'i4',
1411  (parCat['compNGoodStarPerExp'].size, )),
1412  ('COMPSIGFGCM', 'f8',
1413  (parCat['compSigFgcm'].size, )),
1414  ('COMPSIGMACAL', 'f8',
1415  (parCat['compSigmaCal'].size, )),
1416  ('COMPRETRIEVEDLNPWV', 'f8',
1417  (parCat['compRetrievedLnPwv'].size, )),
1418  ('COMPRETRIEVEDLNPWVRAW', 'f8',
1419  (parCat['compRetrievedLnPwvRaw'].size, )),
1420  ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1421  (parCat['compRetrievedLnPwvFlag'].size, )),
1422  ('COMPRETRIEVEDTAUNIGHT', 'f8',
1423  (parCat['compRetrievedTauNight'].size, ))])
1424 
1425  inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1426  inParams['PARO3'][:] = parCat['parO3'][0, :]
1427  inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1428  inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1429  inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1430  inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1431  inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1432  inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1433  inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1434  inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1435  inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1436  inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1437  inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1438  inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1439  inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1440  inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1441  inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1442  inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1443  inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1444  inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1445  inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1446  inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1447  inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1448  inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1449  inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1450  inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1451  inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1452  inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1453  inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1454  inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1455  inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1456  inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1457  inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1458  inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1459  inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1460  inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1461  inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1462  inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1463 
1464  inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1465  inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1466 
1467  return (inParInfo, inParams, inSuperStar)
1468 
1469  def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1470  """
1471  Persist FGCM datasets through the butler.
1472 
1473  Parameters
1474  ----------
1475  fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1476  Fgcm Fit cycle object
1477  """
1478  fgcmDatasetDict = {}
1479 
1480  # Save the parameters
1481  parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1482 
1483  parSchema = afwTable.Schema()
1484 
1485  comma = ','
1486  lutFilterNameString = comma.join([n.decode('utf-8')
1487  for n in parInfo['LUTFILTERNAMES'][0]])
1488  fitBandString = comma.join([n.decode('utf-8')
1489  for n in parInfo['FITBANDS'][0]])
1490 
1491  parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1492  lutFilterNameString, fitBandString)
1493  parCat = self._makeParCatalog(parSchema, parInfo, pars,
1494  fgcmFitCycle.fgcmPars.parSuperStarFlat,
1495  lutFilterNameString, fitBandString)
1496 
1497  fgcmDatasetDict['fgcmFitParameters'] = parCat
1498 
1499  # Save the indices of the flagged stars
1500  # (stars that have been (a) reserved from the fit for testing and
1501  # (b) bad stars that have failed quality checks.)
1502  flagStarSchema = self._makeFlagStarSchema()
1503  flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1504  flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1505 
1506  fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1507 
1508  # Save the zeropoint information and atmospheres only if desired
1509  if self.outputZeropoints:
1510  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1511  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1512 
1513  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1514  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1515 
1516  fgcmDatasetDict['fgcmZeropoints'] = zptCat
1517 
1518  # Save atmosphere values
1519  # These are generated by the same code that generates zeropoints
1520  atmSchema = makeAtmSchema()
1521  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1522 
1523  fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1524 
1525  # Save the standard stars (if configured)
1526  if self.outputStandards:
1527  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1528  stdSchema = makeStdSchema(len(goodBands))
1529  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1530 
1531  fgcmDatasetDict['fgcmStandardStars'] = stdCat
1532 
1533  return fgcmDatasetDict
1534 
1535  def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1536  lutFilterNameString, fitBandString):
1537  """
1538  Make the parameter persistence schema
1539 
1540  Parameters
1541  ----------
1542  parInfo: `numpy.ndarray`
1543  Parameter information returned by fgcm
1544  pars: `numpy.ndarray`
1545  Parameter values returned by fgcm
1546  parSuperStarFlat: `numpy.array`
1547  Superstar flat values returned by fgcm
1548  lutFilterNameString: `str`
1549  Combined string of all the lutFilterNames
1550  fitBandString: `str`
1551  Combined string of all the fitBands
1552 
1553  Returns
1554  -------
1555  parSchema: `afwTable.schema`
1556  """
1557 
1558  parSchema = afwTable.Schema()
1559 
1560  # parameter info section
1561  parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1562  parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1563  size=len(lutFilterNameString))
1564  parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1565  size=len(fitBandString))
1566  parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1567  parSchema.addField('lnTauSlopeUnit', type=np.float64,
1568  doc='Step units for ln(AOD) slope')
1569  parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1570  parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1571  parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1572  doc='Step units for ln(pwv) slope')
1573  parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1574  doc='Step units for ln(pwv) quadratic term')
1575  parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1576  doc='Step units for global ln(pwv) parameters')
1577  parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1578  parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1579  parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1580  parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1581  parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1582 
1583  # parameter section
1584  parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1585  size=pars['PARALPHA'].size)
1586  parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1587  size=pars['PARO3'].size)
1588  parSchema.addField('parLnTauIntercept', type='ArrayD',
1589  doc='ln(Tau) intercept parameter vector',
1590  size=pars['PARLNTAUINTERCEPT'].size)
1591  parSchema.addField('parLnTauSlope', type='ArrayD',
1592  doc='ln(Tau) slope parameter vector',
1593  size=pars['PARLNTAUSLOPE'].size)
1594  parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1595  size=pars['PARLNPWVINTERCEPT'].size)
1596  parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1597  size=pars['PARLNPWVSLOPE'].size)
1598  parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1599  size=pars['PARLNPWVQUADRATIC'].size)
1600  parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1601  size=pars['PARQESYSINTERCEPT'].size)
1602  parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1603  size=pars[0]['COMPQESYSSLOPE'].size)
1604  parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1605  size=pars['PARFILTEROFFSET'].size)
1606  parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1607  size=pars['PARFILTEROFFSETFITFLAG'].size)
1608  parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1609  doc='Global scale for retrieved ln(pwv)')
1610  parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1611  doc='Global offset for retrieved ln(pwv)')
1612  parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1613  doc='Nightly offset for retrieved ln(pwv)',
1614  size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1615  parSchema.addField('compAbsThroughput', type='ArrayD',
1616  doc='Absolute throughput (relative to transmission curves)',
1617  size=pars['COMPABSTHROUGHPUT'].size)
1618  parSchema.addField('compRefOffset', type='ArrayD',
1619  doc='Offset between reference stars and calibrated stars',
1620  size=pars['COMPREFOFFSET'].size)
1621  parSchema.addField('compRefSigma', type='ArrayD',
1622  doc='Width of reference star/calibrated star distribution',
1623  size=pars['COMPREFSIGMA'].size)
1624  parSchema.addField('compMirrorChromaticity', type='ArrayD',
1625  doc='Computed mirror chromaticity terms',
1626  size=pars['COMPMIRRORCHROMATICITY'].size)
1627  parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1628  doc='Mirror chromaticity pivot mjd',
1629  size=pars['MIRRORCHROMATICITYPIVOT'].size)
1630  parSchema.addField('compMedianSedSlope', type='ArrayD',
1631  doc='Computed median SED slope (per band)',
1632  size=pars['COMPMEDIANSEDSLOPE'].size)
1633  parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1634  size=pars['COMPAPERCORRPIVOT'].size)
1635  parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1636  size=pars['COMPAPERCORRSLOPE'].size)
1637  parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1638  size=pars['COMPAPERCORRSLOPEERR'].size)
1639  parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1640  size=pars['COMPAPERCORRRANGE'].size)
1641  parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1642  size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1643  parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1644  size=pars['COMPMODELERRFWHMPIVOT'].size)
1645  parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1646  size=pars['COMPMODELERRSKYPIVOT'].size)
1647  parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1648  size=pars['COMPMODELERRPARS'].size)
1649  parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1650  size=pars['COMPEXPGRAY'].size)
1651  parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1652  size=pars['COMPVARGRAY'].size)
1653  parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1654  doc='Computed exposure offset due to background',
1655  size=pars['COMPEXPDELTAMAGBKG'].size)
1656  parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1657  doc='Computed number of good stars per exposure',
1658  size=pars['COMPNGOODSTARPEREXP'].size)
1659  parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1660  size=pars['COMPSIGFGCM'].size)
1661  parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1662  size=pars['COMPSIGMACAL'].size)
1663  parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1664  size=pars['COMPRETRIEVEDLNPWV'].size)
1665  parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1666  size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1667  parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1668  size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1669  parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1670  size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1671  # superstarflat section
1672  parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1673  size=4)
1674  parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1675  size=parSuperStarFlat.size)
1676 
1677  return parSchema
1678 
1679  def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1680  lutFilterNameString, fitBandString):
1681  """
1682  Make the FGCM parameter catalog for persistence
1683 
1684  Parameters
1685  ----------
1686  parSchema: `lsst.afw.table.Schema`
1687  Parameter catalog schema
1688  pars: `numpy.ndarray`
1689  FGCM parameters to put into parCat
1690  parSuperStarFlat: `numpy.array`
1691  FGCM superstar flat array to put into parCat
1692  lutFilterNameString: `str`
1693  Combined string of all the lutFilterNames
1694  fitBandString: `str`
1695  Combined string of all the fitBands
1696 
1697  Returns
1698  -------
1699  parCat: `afwTable.BasicCatalog`
1700  Atmosphere and instrumental model parameter catalog for persistence
1701  """
1702 
1703  parCat = afwTable.BaseCatalog(parSchema)
1704  parCat.reserve(1)
1705 
1706  # The parameter catalog just has one row, with many columns for all the
1707  # atmosphere and instrument fit parameters
1708  rec = parCat.addNew()
1709 
1710  # info section
1711  rec['nCcd'] = parInfo['NCCD']
1712  rec['lutFilterNames'] = lutFilterNameString
1713  rec['fitBands'] = fitBandString
1714  # note these are not currently supported here.
1715  rec['hasExternalPwv'] = 0
1716  rec['hasExternalTau'] = 0
1717 
1718  # parameter section
1719 
1720  scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1721 
1722  arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1723  'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1724  'parQeSysIntercept', 'compQeSysSlope',
1725  'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1726  'parFilterOffset', 'parFilterOffsetFitFlag',
1727  'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1728  'compMirrorChromaticity', 'mirrorChromaticityPivot',
1729  'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1730  'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1731  'compModelErrSkyPivot', 'compModelErrPars',
1732  'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1733  'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1734  'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1735  'compRetrievedTauNight']
1736 
1737  for scalarName in scalarNames:
1738  rec[scalarName] = pars[scalarName.upper()]
1739 
1740  for arrName in arrNames:
1741  rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1742 
1743  # superstar section
1744  rec['superstarSize'][:] = parSuperStarFlat.shape
1745  rec['superstar'][:] = parSuperStarFlat.flatten()
1746 
1747  return parCat
1748 
1749  def _makeFlagStarSchema(self):
1750  """
1751  Make the flagged-stars schema
1752 
1753  Returns
1754  -------
1755  flagStarSchema: `lsst.afw.table.Schema`
1756  """
1757 
1758  flagStarSchema = afwTable.Schema()
1759 
1760  flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1761  flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1762 
1763  return flagStarSchema
1764 
1765  def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1766  """
1767  Make the flagged star catalog for persistence
1768 
1769  Parameters
1770  ----------
1771  flagStarSchema: `lsst.afw.table.Schema`
1772  Flagged star schema
1773  flagStarStruct: `numpy.ndarray`
1774  Flagged star structure from fgcm
1775 
1776  Returns
1777  -------
1778  flagStarCat: `lsst.afw.table.BaseCatalog`
1779  Flagged star catalog for persistence
1780  """
1781 
1782  flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1783  flagStarCat.resize(flagStarStruct.size)
1784 
1785  flagStarCat['objId'][:] = flagStarStruct['OBJID']
1786  flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1787 
1788  return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:874
def makeStdSchema(nBands)
Definition: utilities.py:729
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:49
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:696
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:208
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:617
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:761
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:531
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:390
def translateVisitCatalog(visitCat)
Definition: utilities.py:339