lsst.fgcmcal  21.0.0-5-gb155db7+58d9d10622
fgcmFitCycle.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Perform a single fit cycle of FGCM.
24 
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
28 for the global fit).
29 
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
34 """
35 
36 import sys
37 import traceback
38 import copy
39 
40 import numpy as np
41 
42 import lsst.pex.config as pexConfig
43 import lsst.pipe.base as pipeBase
44 from lsst.pipe.base import connectionTypes
45 import lsst.afw.table as afwTable
46 
47 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
48 from .utilities import extractReferenceMags
49 from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
50 from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51 from .sedterms import SedboundarytermDict, SedtermDict
52 from .utilities import lookupStaticCalibrations
53 
54 import fgcm
55 
56 __all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57 
58 
59 class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
60  dimensions=("instrument",),
61  defaultTemplates={"previousCycleNumber": "-1",
62  "cycleNumber": "0"}):
63  camera = connectionTypes.PrerequisiteInput(
64  doc="Camera instrument",
65  name="camera",
66  storageClass="Camera",
67  dimensions=("instrument",),
68  lookupFunction=lookupStaticCalibrations,
69  isCalibration=True,
70  )
71 
72  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
73  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
74  "chromatic corrections."),
75  name="fgcmLookUpTable",
76  storageClass="Catalog",
77  dimensions=("instrument",),
78  deferLoad=True,
79  )
80 
81  fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
82  doc="Catalog of visit information for fgcm",
83  name="fgcmVisitCatalog",
84  storageClass="Catalog",
85  dimensions=("instrument",),
86  deferLoad=True,
87  )
88 
89  fgcmStarObservations = connectionTypes.PrerequisiteInput(
90  doc="Catalog of star observations for fgcm",
91  name="fgcmStarObservations",
92  storageClass="Catalog",
93  dimensions=("instrument",),
94  deferLoad=True,
95  )
96 
97  fgcmStarIds = connectionTypes.PrerequisiteInput(
98  doc="Catalog of fgcm calibration star IDs",
99  name="fgcmStarIds",
100  storageClass="Catalog",
101  dimensions=("instrument",),
102  deferLoad=True,
103  )
104 
105  fgcmStarIndices = connectionTypes.PrerequisiteInput(
106  doc="Catalog of fgcm calibration star indices",
107  name="fgcmStarIndices",
108  storageClass="Catalog",
109  dimensions=("instrument",),
110  deferLoad=True,
111  )
112 
113  fgcmReferenceStars = connectionTypes.PrerequisiteInput(
114  doc="Catalog of fgcm-matched reference stars",
115  name="fgcmReferenceStars",
116  storageClass="Catalog",
117  dimensions=("instrument",),
118  deferLoad=True,
119  )
120 
121  fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
122  doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
123  name="fgcmFlaggedStars{previousCycleNumber}",
124  storageClass="Catalog",
125  dimensions=("instrument",),
126  deferLoad=True,
127  )
128 
129  fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
130  doc="Catalog of fgcm fit parameters from previous fit cycle",
131  name="fgcmFitParameters{previousCycleNumber}",
132  storageClass="Catalog",
133  dimensions=("instrument",),
134  deferLoad=True,
135  )
136 
137  fgcmFitParameters = connectionTypes.Output(
138  doc="Catalog of fgcm fit parameters from current fit cycle",
139  name="fgcmFitParameters{cycleNumber}",
140  storageClass="Catalog",
141  dimensions=("instrument",),
142  )
143 
144  fgcmFlaggedStars = connectionTypes.Output(
145  doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
146  name="fgcmFlaggedStars{cycleNumber}",
147  storageClass="Catalog",
148  dimensions=("instrument",),
149  )
150 
151  fgcmZeropoints = connectionTypes.Output(
152  doc="Catalog of fgcm zeropoint data from current fit cycle",
153  name="fgcmZeropoints{cycleNumber}",
154  storageClass="Catalog",
155  dimensions=("instrument",),
156  )
157 
158  fgcmAtmosphereParameters = connectionTypes.Output(
159  doc="Catalog of atmospheric fit parameters from current fit cycle",
160  name="fgcmAtmosphereParameters{cycleNumber}",
161  storageClass="Catalog",
162  dimensions=("instrument",),
163  )
164 
165  fgcmStandardStars = connectionTypes.Output(
166  doc="Catalog of standard star magnitudes from current fit cycle",
167  name="fgcmStandardStars{cycleNumber}",
168  storageClass="SimpleCatalog",
169  dimensions=("instrument",),
170  )
171 
172  def __init__(self, *, config=None):
173  super().__init__(config=config)
174 
175  if not config.doReferenceCalibration:
176  self.prerequisiteInputs.remove("fgcmReferenceStars")
177 
178  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
179  raise ValueError("cycleNumber must be of integer format")
180  if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
181  raise ValueError("previousCycleNumber must be of integer format")
182  if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
183  raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
184 
185  if int(config.connections.cycleNumber) == 0:
186  self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
187  self.prerequisiteInputs.remove("fgcmFitParametersInput")
188 
189  if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
190  self.outputs.remove("fgcmStandardStars")
191 
192  if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
193  self.outputs.remove("fgcmZeropoints")
194  self.outputs.remove("fgcmAtmosphereParameters")
195 
196 
197 class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
198  pipelineConnections=FgcmFitCycleConnections):
199  """Config for FgcmFitCycle"""
200 
201  bands = pexConfig.ListField(
202  doc="Bands to run calibration",
203  dtype=str,
204  default=[],
205  )
206  fitFlag = pexConfig.ListField(
207  doc=("Flag for which bands are directly constrained in the FGCM fit. "
208  "Bands set to 0 will have the atmosphere constrained from observations "
209  "in other bands on the same night. Must be same length as config.bands, "
210  "and matched band-by-band."),
211  dtype=int,
212  default=(0,),
213  optional=True,
214  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
215  "It will be removed after v20. Use fitBands instead."),
216  )
217  fitBands = pexConfig.ListField(
218  doc=("Bands to use in atmospheric fit. The bands not listed here will have "
219  "the atmosphere constrained from the 'fitBands' on the same night. "
220  "Must be a subset of `config.bands`"),
221  dtype=str,
222  default=[],
223  )
224  requiredFlag = pexConfig.ListField(
225  doc=("Flag for which bands are required for a star to be considered a calibration "
226  "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
227  "be same length as config.bands, and matched band-by-band."),
228  dtype=int,
229  default=(0,),
230  optional=True,
231  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
232  "It will be removed after v20. Use requiredBands instead."),
233  )
234  requiredBands = pexConfig.ListField(
235  doc=("Bands that are required for a star to be considered a calibration star. "
236  "Must be a subset of `config.bands`"),
237  dtype=str,
238  default=[],
239  )
240  filterMap = pexConfig.DictField(
241  doc="Mapping from 'filterName' to band.",
242  keytype=str,
243  itemtype=str,
244  default={},
245  )
246  doReferenceCalibration = pexConfig.Field(
247  doc="Use reference catalog as additional constraint on calibration",
248  dtype=bool,
249  default=True,
250  )
251  refStarSnMin = pexConfig.Field(
252  doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
253  dtype=float,
254  default=50.0,
255  )
256  refStarOutlierNSig = pexConfig.Field(
257  doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
258  "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
259  dtype=float,
260  default=4.0,
261  )
262  applyRefStarColorCuts = pexConfig.Field(
263  doc="Apply color cuts to reference stars?",
264  dtype=bool,
265  default=True,
266  )
267  nCore = pexConfig.Field(
268  doc="Number of cores to use",
269  dtype=int,
270  default=4,
271  )
272  nStarPerRun = pexConfig.Field(
273  doc="Number of stars to run in each chunk",
274  dtype=int,
275  default=200000,
276  )
277  nExpPerRun = pexConfig.Field(
278  doc="Number of exposures to run in each chunk",
279  dtype=int,
280  default=1000,
281  )
282  reserveFraction = pexConfig.Field(
283  doc="Fraction of stars to reserve for testing",
284  dtype=float,
285  default=0.1,
286  )
287  freezeStdAtmosphere = pexConfig.Field(
288  doc="Freeze atmosphere parameters to standard (for testing)",
289  dtype=bool,
290  default=False,
291  )
292  precomputeSuperStarInitialCycle = pexConfig.Field(
293  doc="Precompute superstar flat for initial cycle",
294  dtype=bool,
295  default=False,
296  )
297  superStarSubCcd = pexConfig.Field(
298  doc="Compute superstar flat on sub-ccd scale",
299  dtype=bool,
300  default=True,
301  optional=True,
302  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
303  "It will be removed after v20. Use superStarSubCcdDict instead."),
304  )
305  superStarSubCcdDict = pexConfig.DictField(
306  doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
307  "Must have one entry per band."),
308  keytype=str,
309  itemtype=bool,
310  default={},
311  )
312  superStarSubCcdChebyshevOrder = pexConfig.Field(
313  doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
314  "Global default is first-order polynomials, and should be overridden "
315  "on a camera-by-camera basis depending on the ISR."),
316  dtype=int,
317  default=1,
318  )
319  superStarSubCcdTriangular = pexConfig.Field(
320  doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
321  "suppress high-order cross terms?"),
322  dtype=bool,
323  default=False,
324  )
325  superStarSigmaClip = pexConfig.Field(
326  doc="Number of sigma to clip outliers when selecting for superstar flats",
327  dtype=float,
328  default=5.0,
329  )
330  focalPlaneSigmaClip = pexConfig.Field(
331  doc="Number of sigma to clip outliers per focal-plane.",
332  dtype=float,
333  default=4.0,
334  )
335  ccdGraySubCcd = pexConfig.Field(
336  doc="Compute CCD gray terms on sub-ccd scale",
337  dtype=bool,
338  default=False,
339  optional=True,
340  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
341  "It will be removed after v20. Use ccdGraySubCcdDict instead."),
342  )
343  ccdGraySubCcdDict = pexConfig.DictField(
344  doc=("Per-band specification on whether to compute achromatic per-ccd residual "
345  "('ccd gray') on a sub-ccd scale."),
346  keytype=str,
347  itemtype=bool,
348  default={},
349  )
350  ccdGraySubCcdChebyshevOrder = pexConfig.Field(
351  doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
352  dtype=int,
353  default=1,
354  )
355  ccdGraySubCcdTriangular = pexConfig.Field(
356  doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
357  "suppress high-order cross terms?"),
358  dtype=bool,
359  default=True,
360  )
361  ccdGrayFocalPlaneDict = pexConfig.DictField(
362  doc=("Per-band specification on whether to compute focal-plane residual "
363  "('ccd gray') corrections."),
364  keytype=str,
365  itemtype=bool,
366  default={},
367  )
368  ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
369  doc=("Minimum number of 'good' CCDs required to perform focal-plane "
370  "gray corrections. If there are fewer good CCDs then the gray "
371  "correction is computed per-ccd."),
372  dtype=int,
373  default=1,
374  )
375  ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
376  doc="Order of the 2D chebyshev polynomials for focal plane fit.",
377  dtype=int,
378  default=3,
379  )
380  cycleNumber = pexConfig.Field(
381  doc=("FGCM fit cycle number. This is automatically incremented after each run "
382  "and stage of outlier rejection. See cookbook for details."),
383  dtype=int,
384  default=None,
385  )
386  isFinalCycle = pexConfig.Field(
387  doc=("Is this the final cycle of the fitting? Will automatically compute final "
388  "selection of stars and photometric exposures, and will output zeropoints "
389  "and standard stars for use in fgcmOutputProducts"),
390  dtype=bool,
391  default=False,
392  )
393  maxIterBeforeFinalCycle = pexConfig.Field(
394  doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
395  "will always be 0 in the final cycle for cleanup and final selection."),
396  dtype=int,
397  default=50,
398  )
399  deltaMagBkgOffsetPercentile = pexConfig.Field(
400  doc=("Percentile brightest stars on a visit/ccd to use to compute net "
401  "offset from local background subtraction."),
402  dtype=float,
403  default=0.25,
404  )
405  deltaMagBkgPerCcd = pexConfig.Field(
406  doc=("Compute net offset from local background subtraction per-ccd? "
407  "Otherwise, use computation per visit."),
408  dtype=bool,
409  default=False,
410  )
411  utBoundary = pexConfig.Field(
412  doc="Boundary (in UTC) from day-to-day",
413  dtype=float,
414  default=None,
415  )
416  washMjds = pexConfig.ListField(
417  doc="Mirror wash MJDs",
418  dtype=float,
419  default=(0.0,),
420  )
421  epochMjds = pexConfig.ListField(
422  doc="Epoch boundaries in MJD",
423  dtype=float,
424  default=(0.0,),
425  )
426  minObsPerBand = pexConfig.Field(
427  doc="Minimum good observations per band",
428  dtype=int,
429  default=2,
430  )
431  # TODO: When DM-16511 is done, it will be possible to get the
432  # telescope latitude directly from the camera.
433  latitude = pexConfig.Field(
434  doc="Observatory latitude",
435  dtype=float,
436  default=None,
437  )
438  brightObsGrayMax = pexConfig.Field(
439  doc="Maximum gray extinction to be considered bright observation",
440  dtype=float,
441  default=0.15,
442  )
443  minStarPerCcd = pexConfig.Field(
444  doc=("Minimum number of good stars per CCD to be used in calibration fit. "
445  "CCDs with fewer stars will have their calibration estimated from other "
446  "CCDs in the same visit, with zeropoint error increased accordingly."),
447  dtype=int,
448  default=5,
449  )
450  minCcdPerExp = pexConfig.Field(
451  doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
452  "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
453  dtype=int,
454  default=5,
455  )
456  maxCcdGrayErr = pexConfig.Field(
457  doc="Maximum error on CCD gray offset to be considered photometric",
458  dtype=float,
459  default=0.05,
460  )
461  minStarPerExp = pexConfig.Field(
462  doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
463  "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
464  dtype=int,
465  default=600,
466  )
467  minExpPerNight = pexConfig.Field(
468  doc="Minimum number of good exposures/visits to consider a partly photometric night",
469  dtype=int,
470  default=10,
471  )
472  expGrayInitialCut = pexConfig.Field(
473  doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
474  "observations."),
475  dtype=float,
476  default=-0.25,
477  )
478  expGrayPhotometricCut = pexConfig.ListField(
479  doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
480  "Must be same length as config.bands, and matched band-by-band."),
481  dtype=float,
482  default=(0.0,),
483  optional=True,
484  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
485  "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
486  )
487  expGrayPhotometricCutDict = pexConfig.DictField(
488  doc=("Per-band specification on maximum (negative) achromatic exposure residual "
489  "('gray term') for a visit to be considered photometric. Must have one "
490  "entry per band. Broad-band filters should be -0.05."),
491  keytype=str,
492  itemtype=float,
493  default={},
494  )
495  expGrayHighCut = pexConfig.ListField(
496  doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
497  "Must be same length as config.bands, and matched band-by-band."),
498  dtype=float,
499  default=(0.0,),
500  optional=True,
501  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
502  "It will be removed after v20. Use expGrayHighCutDict instead."),
503  )
504  expGrayHighCutDict = pexConfig.DictField(
505  doc=("Per-band specification on maximum (positive) achromatic exposure residual "
506  "('gray term') for a visit to be considered photometric. Must have one "
507  "entry per band. Broad-band filters should be 0.2."),
508  keytype=str,
509  itemtype=float,
510  default={},
511  )
512  expGrayRecoverCut = pexConfig.Field(
513  doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
514  "Visits with more gray extinction will only get CCD zeropoints if there are "
515  "sufficient star observations (minStarPerCcd) on that CCD."),
516  dtype=float,
517  default=-1.0,
518  )
519  expVarGrayPhotometricCut = pexConfig.Field(
520  doc="Maximum exposure variance to be considered possibly photometric",
521  dtype=float,
522  default=0.0005,
523  optional=True,
524  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
525  "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
526  )
527  expVarGrayPhotometricCutDict = pexConfig.DictField(
528  doc=("Per-band specification on maximum exposure variance to be considered possibly "
529  "photometric. Must have one entry per band. Broad-band filters should be "
530  "0.0005."),
531  keytype=str,
532  itemtype=float,
533  default={},
534  )
535  expGrayErrRecoverCut = pexConfig.Field(
536  doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
537  "Visits with more gray variance will only get CCD zeropoints if there are "
538  "sufficient star observations (minStarPerCcd) on that CCD."),
539  dtype=float,
540  default=0.05,
541  )
542  aperCorrFitNBins = pexConfig.Field(
543  doc=("Number of aperture bins used in aperture correction fit. When set to 0"
544  "no fit will be performed, and the config.aperCorrInputSlopes will be "
545  "used if available."),
546  dtype=int,
547  default=10,
548  )
549  aperCorrInputSlopes = pexConfig.ListField(
550  doc=("Aperture correction input slope parameters. These are used on the first "
551  "fit iteration, and aperture correction parameters will be updated from "
552  "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
553  "when there is insufficient data to fit the parameters (e.g. tract mode). "
554  "If set, must be same length as config.bands, and matched band-by-band."),
555  dtype=float,
556  default=[],
557  optional=True,
558  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
559  "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
560  )
561  aperCorrInputSlopeDict = pexConfig.DictField(
562  doc=("Per-band specification of aperture correction input slope parameters. These "
563  "are used on the first fit iteration, and aperture correction parameters will "
564  "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
565  "to set this when there is insufficient data to fit the parameters (e.g. "
566  "tract mode)."),
567  keytype=str,
568  itemtype=float,
569  default={},
570  )
571  sedFudgeFactors = pexConfig.ListField(
572  doc=("Fudge factors for computing linear SED from colors. Must be same length as "
573  "config.bands, and matched band-by-band."),
574  dtype=float,
575  default=(0,),
576  optional=True,
577  deprecated=("This field has been deprecated and will be removed after v20. "
578  "Please use sedSlopeTermMap and sedSlopeMap."),
579  )
580  sedboundaryterms = pexConfig.ConfigField(
581  doc="Mapping from bands to SED boundary term names used is sedterms.",
582  dtype=SedboundarytermDict,
583  )
584  sedterms = pexConfig.ConfigField(
585  doc="Mapping from terms to bands for fgcm linear SED approximations.",
586  dtype=SedtermDict,
587  )
588  sigFgcmMaxErr = pexConfig.Field(
589  doc="Maximum mag error for fitting sigma_FGCM",
590  dtype=float,
591  default=0.01,
592  )
593  sigFgcmMaxEGray = pexConfig.ListField(
594  doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
595  "May be 1 element (same for all bands) or the same length as config.bands."),
596  dtype=float,
597  default=(0.05,),
598  optional=True,
599  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
600  "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
601  )
602  sigFgcmMaxEGrayDict = pexConfig.DictField(
603  doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
604  "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
605  "should be 0.05."),
606  keytype=str,
607  itemtype=float,
608  default={},
609  )
610  ccdGrayMaxStarErr = pexConfig.Field(
611  doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
612  "computation"),
613  dtype=float,
614  default=0.10,
615  )
616  approxThroughput = pexConfig.ListField(
617  doc=("Approximate overall throughput at start of calibration observations. "
618  "May be 1 element (same for all bands) or the same length as config.bands."),
619  dtype=float,
620  default=(1.0, ),
621  optional=True,
622  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
623  "It will be removed after v20. Use approxThroughputDict instead."),
624  )
625  approxThroughputDict = pexConfig.DictField(
626  doc=("Per-band specification of the approximate overall throughput at the start of "
627  "calibration observations. Must have one entry per band. Typically should "
628  "be 1.0."),
629  keytype=str,
630  itemtype=float,
631  default={},
632  )
633  sigmaCalRange = pexConfig.ListField(
634  doc="Allowed range for systematic error floor estimation",
635  dtype=float,
636  default=(0.001, 0.003),
637  )
638  sigmaCalFitPercentile = pexConfig.ListField(
639  doc="Magnitude percentile range to fit systematic error floor",
640  dtype=float,
641  default=(0.05, 0.15),
642  )
643  sigmaCalPlotPercentile = pexConfig.ListField(
644  doc="Magnitude percentile range to plot systematic error floor",
645  dtype=float,
646  default=(0.05, 0.95),
647  )
648  sigma0Phot = pexConfig.Field(
649  doc="Systematic error floor for all zeropoints",
650  dtype=float,
651  default=0.003,
652  )
653  mapLongitudeRef = pexConfig.Field(
654  doc="Reference longitude for plotting maps",
655  dtype=float,
656  default=0.0,
657  )
658  mapNSide = pexConfig.Field(
659  doc="Healpix nside for plotting maps",
660  dtype=int,
661  default=256,
662  )
663  outfileBase = pexConfig.Field(
664  doc="Filename start for plot output files",
665  dtype=str,
666  default=None,
667  )
668  starColorCuts = pexConfig.ListField(
669  doc="Encoded star-color cuts (to be cleaned up)",
670  dtype=str,
671  default=("NO_DATA",),
672  )
673  colorSplitIndices = pexConfig.ListField(
674  doc="Band indices to use to split stars by color",
675  dtype=int,
676  default=None,
677  optional=True,
678  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
679  "It will be removed after v20. Use colorSplitBands instead."),
680  )
681  colorSplitBands = pexConfig.ListField(
682  doc="Band names to use to split stars by color. Must have 2 entries.",
683  dtype=str,
684  length=2,
685  default=('g', 'i'),
686  )
687  modelMagErrors = pexConfig.Field(
688  doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
689  dtype=bool,
690  default=True,
691  )
692  useQuadraticPwv = pexConfig.Field(
693  doc="Model PWV with a quadratic term for variation through the night?",
694  dtype=bool,
695  default=False,
696  )
697  instrumentParsPerBand = pexConfig.Field(
698  doc=("Model instrumental parameters per band? "
699  "Otherwise, instrumental parameters (QE changes with time) are "
700  "shared among all bands."),
701  dtype=bool,
702  default=False,
703  )
704  instrumentSlopeMinDeltaT = pexConfig.Field(
705  doc=("Minimum time change (in days) between observations to use in constraining "
706  "instrument slope."),
707  dtype=float,
708  default=20.0,
709  )
710  fitMirrorChromaticity = pexConfig.Field(
711  doc="Fit (intraband) mirror chromatic term?",
712  dtype=bool,
713  default=False,
714  )
715  coatingMjds = pexConfig.ListField(
716  doc="Mirror coating dates in MJD",
717  dtype=float,
718  default=(0.0,),
719  )
720  outputStandardsBeforeFinalCycle = pexConfig.Field(
721  doc="Output standard stars prior to final cycle? Used in debugging.",
722  dtype=bool,
723  default=False,
724  )
725  outputZeropointsBeforeFinalCycle = pexConfig.Field(
726  doc="Output standard stars prior to final cycle? Used in debugging.",
727  dtype=bool,
728  default=False,
729  )
730  useRepeatabilityForExpGrayCuts = pexConfig.ListField(
731  doc=("Use star repeatability (instead of exposures) for computing photometric "
732  "cuts? Recommended for tract mode or bands with few exposures. "
733  "May be 1 element (same for all bands) or the same length as config.bands."),
734  dtype=bool,
735  default=(False,),
736  optional=True,
737  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
738  "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
739  )
740  useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
741  doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
742  "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
743  keytype=str,
744  itemtype=bool,
745  default={},
746  )
747  autoPhotometricCutNSig = pexConfig.Field(
748  doc=("Number of sigma for automatic computation of (low) photometric cut. "
749  "Cut is based on exposure gray width (per band), unless "
750  "useRepeatabilityForExpGrayCuts is set, in which case the star "
751  "repeatability is used (also per band)."),
752  dtype=float,
753  default=3.0,
754  )
755  autoHighCutNSig = pexConfig.Field(
756  doc=("Number of sigma for automatic computation of (high) outlier cut. "
757  "Cut is based on exposure gray width (per band), unless "
758  "useRepeatabilityForExpGrayCuts is set, in which case the star "
759  "repeatability is used (also per band)."),
760  dtype=float,
761  default=4.0,
762  )
763  quietMode = pexConfig.Field(
764  doc="Be less verbose with logging.",
765  dtype=bool,
766  default=False,
767  )
768  randomSeed = pexConfig.Field(
769  doc="Random seed for fgcm for consistency in tests.",
770  dtype=int,
771  default=None,
772  optional=True,
773  )
774 
775  def validate(self):
776  super().validate()
777 
778  if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
779  msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
780  raise RuntimeError(msg)
781  if self.connections.cycleNumber != str(self.cycleNumber):
782  msg = "cycleNumber in template must be equal to connections.cycleNumber"
783  raise RuntimeError(msg)
784 
785  for band in self.fitBands:
786  if band not in self.bands:
787  msg = 'fitBand %s not in bands' % (band)
788  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
789  for band in self.requiredBands:
790  if band not in self.bands:
791  msg = 'requiredBand %s not in bands' % (band)
792  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
793  for band in self.colorSplitBands:
794  if band not in self.bands:
795  msg = 'colorSplitBand %s not in bands' % (band)
796  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
797  for band in self.bands:
798  if band not in self.superStarSubCcdDict:
799  msg = 'band %s not in superStarSubCcdDict' % (band)
800  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
801  self, msg)
802  if band not in self.ccdGraySubCcdDict:
803  msg = 'band %s not in ccdGraySubCcdDict' % (band)
804  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
805  self, msg)
806  if band not in self.expGrayPhotometricCutDict:
807  msg = 'band %s not in expGrayPhotometricCutDict' % (band)
808  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
809  self, msg)
810  if band not in self.expGrayHighCutDict:
811  msg = 'band %s not in expGrayHighCutDict' % (band)
812  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
813  self, msg)
814  if band not in self.expVarGrayPhotometricCutDict:
815  msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
816  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
817  self, msg)
818  if band not in self.sigFgcmMaxEGrayDict:
819  msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
820  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
821  self, msg)
822  if band not in self.approxThroughputDict:
823  msg = 'band %s not in approxThroughputDict' % (band)
824  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
825  self, msg)
826  if band not in self.useRepeatabilityForExpGrayCutsDict:
827  msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
828  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
829  self, msg)
830 
831 
832 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
833  """Subclass of TaskRunner for fgcmFitCycleTask
834 
835  fgcmFitCycleTask.run() takes one argument, the butler, and uses
836  stars and visits previously extracted from dataRefs by
837  fgcmBuildStars.
838  This Runner does not perform any dataRef parallelization, but the FGCM
839  code called by the Task uses python multiprocessing (see the "ncores"
840  config option).
841  """
842 
843  @staticmethod
844  def getTargetList(parsedCmd):
845  """
846  Return a list with one element, the butler.
847  """
848  return [parsedCmd.butler]
849 
850  def __call__(self, butler):
851  """
852  Parameters
853  ----------
854  butler: `lsst.daf.persistence.Butler`
855 
856  Returns
857  -------
858  exitStatus: `list` with `pipeBase.Struct`
859  exitStatus (0: success; 1: failure)
860  """
861 
862  task = self.TaskClass(config=self.config, log=self.log)
863 
864  exitStatus = 0
865  if self.doRaise:
866  task.runDataRef(butler)
867  else:
868  try:
869  task.runDataRef(butler)
870  except Exception as e:
871  exitStatus = 1
872  task.log.fatal("Failed: %s" % e)
873  if not isinstance(e, pipeBase.TaskError):
874  traceback.print_exc(file=sys.stderr)
875 
876  task.writeMetadata(butler)
877 
878  # The task does not return any results:
879  return [pipeBase.Struct(exitStatus=exitStatus)]
880 
881  def run(self, parsedCmd):
882  """
883  Run the task, with no multiprocessing
884 
885  Parameters
886  ----------
887  parsedCmd: ArgumentParser parsed command line
888  """
889 
890  resultList = []
891 
892  if self.precall(parsedCmd):
893  targetList = self.getTargetList(parsedCmd)
894  # make sure that we only get 1
895  resultList = self(targetList[0])
896 
897  return resultList
898 
899 
900 class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
901  """
902  Run Single fit cycle for FGCM global calibration
903  """
904 
905  ConfigClass = FgcmFitCycleConfig
906  RunnerClass = FgcmFitCycleRunner
907  _DefaultName = "fgcmFitCycle"
908 
909  def __init__(self, butler=None, initInputs=None, **kwargs):
910  super().__init__(**kwargs)
911 
912  # no saving of metadata for now
913  def _getMetadataName(self):
914  return None
915 
916  def runQuantum(self, butlerQC, inputRefs, outputRefs):
917  camera = butlerQC.get(inputRefs.camera)
918 
919  dataRefDict = {}
920 
921  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
922  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
923  dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
924  dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
925  dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
926  if self.config.doReferenceCalibration:
927  dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
928  if self.config.cycleNumber > 0:
929  dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
930  dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
931 
932  fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
933 
934  butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
935  butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
936  if self.outputZeropoints:
937  butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
938  butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
939  if self.outputStandards:
940  butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
941 
942  @pipeBase.timeMethod
943  def runDataRef(self, butler):
944  """
945  Run a single fit cycle for FGCM
946 
947  Parameters
948  ----------
949  butler: `lsst.daf.persistence.Butler`
950  """
951  self._checkDatasetsExist(butler)
952 
953  dataRefDict = {}
954  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
955  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
956  dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
957  dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
958  dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
959  if self.config.doReferenceCalibration:
960  dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
961  if self.config.cycleNumber > 0:
962  lastCycle = self.config.cycleNumber - 1
963  dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
964  fgcmcycle=lastCycle)
965  dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
966  fgcmcycle=lastCycle)
967 
968  camera = butler.get('camera')
969  fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
970 
971  butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
972  fgcmcycle=self.config.cycleNumber)
973  butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
974  fgcmcycle=self.config.cycleNumber)
975  if self.outputZeropoints:
976  butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
977  fgcmcycle=self.config.cycleNumber)
978  butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
979  fgcmcycle=self.config.cycleNumber)
980  if self.outputStandards:
981  butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
982  fgcmcycle=self.config.cycleNumber)
983 
984  def writeConfig(self, butler, clobber=False, doBackup=True):
985  """Write the configuration used for processing the data, or check that an existing
986  one is equal to the new one if present. This is an override of the regular
987  version from pipe_base that knows about fgcmcycle.
988 
989  Parameters
990  ----------
991  butler : `lsst.daf.persistence.Butler`
992  Data butler used to write the config. The config is written to dataset type
993  `CmdLineTask._getConfigName`.
994  clobber : `bool`, optional
995  A boolean flag that controls what happens if a config already has been saved:
996  - `True`: overwrite or rename the existing config, depending on ``doBackup``.
997  - `False`: raise `TaskError` if this config does not match the existing config.
998  doBackup : `bool`, optional
999  Set to `True` to backup the config files if clobbering.
1000  """
1001  configName = self._getConfigName()
1002  if configName is None:
1003  return
1004  if clobber:
1005  butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1006  elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1007  # this may be subject to a race condition; see #2789
1008  try:
1009  oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1010  except Exception as exc:
1011  raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1012  (configName, exc))
1013 
1014  def logConfigMismatch(msg):
1015  self.log.fatal("Comparing configuration: %s", msg)
1016 
1017  if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1018  raise pipeBase.TaskError(
1019  f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1020  " must be consistent within the same output repo (override with --clobber-config)")
1021  else:
1022  butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1023 
1024  def _fgcmFitCycle(self, camera, dataRefDict):
1025  """
1026  Run the fit cycle
1027 
1028  Parameters
1029  ----------
1030  camera : `lsst.afw.cameraGeom.Camera`
1031  dataRefDict : `dict`
1032  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1033  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1034  dataRef dictionary with keys:
1035 
1036  ``"fgcmLookUpTable"``
1037  dataRef for the FGCM look-up table.
1038  ``"fgcmVisitCatalog"``
1039  dataRef for visit summary catalog.
1040  ``"fgcmStarObservations"``
1041  dataRef for star observation catalog.
1042  ``"fgcmStarIds"``
1043  dataRef for star id catalog.
1044  ``"fgcmStarIndices"``
1045  dataRef for star index catalog.
1046  ``"fgcmReferenceStars"``
1047  dataRef for matched reference star catalog.
1048  ``"fgcmFlaggedStars"``
1049  dataRef for flagged star catalog.
1050  ``"fgcmFitParameters"``
1051  dataRef for fit parameter catalog.
1052 
1053  Returns
1054  -------
1055  fgcmDatasetDict : `dict`
1056  Dictionary of datasets to persist.
1057  """
1058  # Set defaults on whether to output standards and zeropoints
1059  self.maxIter = self.config.maxIterBeforeFinalCycle
1060  self.outputStandards = self.config.outputStandardsBeforeFinalCycle
1061  self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
1062  self.resetFitParameters = True
1063 
1064  if self.config.isFinalCycle:
1065  # This is the final fit cycle, so we do not want to reset fit
1066  # parameters, we want to run a final "clean-up" with 0 fit iterations,
1067  # and we always want to output standards and zeropoints
1068  self.maxIter = 0
1069  self.outputStandards = True
1070  self.outputZeropoints = True
1071  self.resetFitParameters = False
1072 
1073  configDict = makeConfigDict(self.config, self.log, camera,
1074  self.maxIter, self.resetFitParameters,
1075  self.outputZeropoints)
1076 
1077  lutCat = dataRefDict['fgcmLookUpTable'].get()
1078  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
1079  del lutCat
1080 
1081  # next we need the exposure/visit information
1082  visitCat = dataRefDict['fgcmVisitCatalog'].get()
1083  fgcmExpInfo = translateVisitCatalog(visitCat)
1084  del visitCat
1085 
1086  # Use the first orientation.
1087  # TODO: DM-21215 will generalize to arbitrary camera orientations
1088  ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1089 
1090  noFitsDict = {'lutIndex': lutIndexVals,
1091  'lutStd': lutStd,
1092  'expInfo': fgcmExpInfo,
1093  'ccdOffsets': ccdOffsets}
1094 
1095  # set up the fitter object
1096  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1097  noFitsDict=noFitsDict, noOutput=True)
1098 
1099  # create the parameter object
1100  if (fgcmFitCycle.initialCycle):
1101  # cycle = 0, initial cycle
1102  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1103  fgcmLut,
1104  fgcmExpInfo)
1105  else:
1106  inParInfo, inParams, inSuperStar = self._loadParameters(dataRefDict['fgcmFitParameters'])
1107  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1108  fgcmExpInfo,
1109  inParInfo,
1110  inParams,
1111  inSuperStar)
1112 
1113  # set up the stars...
1114  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1115 
1116  starObs = dataRefDict['fgcmStarObservations'].get()
1117  starIds = dataRefDict['fgcmStarIds'].get()
1118  starIndices = dataRefDict['fgcmStarIndices'].get()
1119 
1120  # grab the flagged stars if available
1121  if 'fgcmFlaggedStars' in dataRefDict:
1122  flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1123  flagId = flaggedStars['objId'][:]
1124  flagFlag = flaggedStars['objFlag'][:]
1125  else:
1126  flaggedStars = None
1127  flagId = None
1128  flagFlag = None
1129 
1130  if self.config.doReferenceCalibration:
1131  refStars = dataRefDict['fgcmReferenceStars'].get()
1132 
1133  refMag, refMagErr = extractReferenceMags(refStars,
1134  self.config.bands,
1135  self.config.filterMap)
1136  refId = refStars['fgcm_id'][:]
1137  else:
1138  refStars = None
1139  refId = None
1140  refMag = None
1141  refMagErr = None
1142 
1143  # match star observations to visits
1144  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1145  # actually be transferred into fgcm using the indexing below.
1146  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1147 
1148  # The fgcmStars.loadStars method will copy all the star information into
1149  # special shared memory objects that will not blow up the memory usage when
1150  # used with python multiprocessing. Once all the numbers are copied,
1151  # it is necessary to release all references to the objects that previously
1152  # stored the data to ensure that the garbage collector can clear the memory,
1153  # and ensure that this memory is not copied when multiprocessing kicks in.
1154 
1155  # We determine the conversion from the native units (typically radians) to
1156  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1157  # numpy arrays rather than Angles, which would we approximately 600x slower.
1158  conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1159 
1160  fgcmStars.loadStars(fgcmPars,
1161  starObs['visit'][starIndices['obsIndex']],
1162  starObs['ccd'][starIndices['obsIndex']],
1163  starObs['ra'][starIndices['obsIndex']] * conv,
1164  starObs['dec'][starIndices['obsIndex']] * conv,
1165  starObs['instMag'][starIndices['obsIndex']],
1166  starObs['instMagErr'][starIndices['obsIndex']],
1167  fgcmExpInfo['FILTERNAME'][visitIndex],
1168  starIds['fgcm_id'][:],
1169  starIds['ra'][:],
1170  starIds['dec'][:],
1171  starIds['obsArrIndex'][:],
1172  starIds['nObs'][:],
1173  obsX=starObs['x'][starIndices['obsIndex']],
1174  obsY=starObs['y'][starIndices['obsIndex']],
1175  obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1176  psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1177  refID=refId,
1178  refMag=refMag,
1179  refMagErr=refMagErr,
1180  flagID=flagId,
1181  flagFlag=flagFlag,
1182  computeNobs=True)
1183 
1184  # Release all references to temporary objects holding star data (see above)
1185  del starObs
1186  del starIds
1187  del starIndices
1188  del flagId
1189  del flagFlag
1190  del flaggedStars
1191  del refStars
1192  del refId
1193  del refMag
1194  del refMagErr
1195 
1196  # and set the bits in the cycle object
1197  fgcmFitCycle.setLUT(fgcmLut)
1198  fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1199  fgcmFitCycle.setPars(fgcmPars)
1200 
1201  # finish the setup
1202  fgcmFitCycle.finishSetup()
1203 
1204  # and run
1205  fgcmFitCycle.run()
1206 
1207 
1210 
1211  fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1212 
1213  # Output the config for the next cycle
1214  # We need to make a copy since the input one has been frozen
1215 
1216  updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1217  i, b in enumerate(self.config.bands)}
1218  updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1219  i, band in enumerate(self.config.bands)}
1220 
1221  outConfig = copy.copy(self.config)
1222  outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
1223  precomputeSuperStarInitialCycle=False,
1224  freezeStdAtmosphere=False,
1225  expGrayPhotometricCutDict=updatedPhotometricCutDict,
1226  expGrayHighCutDict=updatedHighCutDict)
1227 
1228  outConfig.connections.update(previousCycleNumber=str(self.config.cycleNumber),
1229  cycleNumber=str(self.config.cycleNumber + 1))
1230 
1231  configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1232  outConfig.cycleNumber)
1233  outConfig.save(configFileName)
1234 
1235  if self.config.isFinalCycle == 1:
1236  # We are done, ready to output products
1237  self.log.info("Everything is in place to run fgcmOutputProducts.py")
1238  else:
1239  self.log.info("Saved config for next cycle to %s" % (configFileName))
1240  self.log.info("Be sure to look at:")
1241  self.log.info(" config.expGrayPhotometricCut")
1242  self.log.info(" config.expGrayHighCut")
1243  self.log.info("If you are satisfied with the fit, please set:")
1244  self.log.info(" config.isFinalCycle = True")
1245 
1246  return fgcmDatasetDict
1247 
1248  def _checkDatasetsExist(self, butler):
1249  """
1250  Check if necessary datasets exist to run fgcmFitCycle
1251 
1252  Parameters
1253  ----------
1254  butler: `lsst.daf.persistence.Butler`
1255 
1256  Raises
1257  ------
1258  RuntimeError
1259  If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1260  fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1261  If cycleNumber > 0, then also checks for fgcmFitParameters,
1262  fgcmFlaggedStars.
1263  """
1264 
1265  if not butler.datasetExists('fgcmVisitCatalog'):
1266  raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1267  if not butler.datasetExists('fgcmStarObservations'):
1268  raise RuntimeError("Could not find fgcmStarObservations in repo!")
1269  if not butler.datasetExists('fgcmStarIds'):
1270  raise RuntimeError("Could not find fgcmStarIds in repo!")
1271  if not butler.datasetExists('fgcmStarIndices'):
1272  raise RuntimeError("Could not find fgcmStarIndices in repo!")
1273  if not butler.datasetExists('fgcmLookUpTable'):
1274  raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1275 
1276  # Need additional datasets if we are not the initial cycle
1277  if (self.config.cycleNumber > 0):
1278  if not butler.datasetExists('fgcmFitParameters',
1279  fgcmcycle=self.config.cycleNumber-1):
1280  raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1281  (self.config.cycleNumber-1))
1282  if not butler.datasetExists('fgcmFlaggedStars',
1283  fgcmcycle=self.config.cycleNumber-1):
1284  raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1285  (self.config.cycleNumber-1))
1286 
1287  # And additional dataset if we want reference calibration
1288  if self.config.doReferenceCalibration:
1289  if not butler.datasetExists('fgcmReferenceStars'):
1290  raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1291  "doReferenceCalibration is True.")
1292 
1293  def _loadParameters(self, parDataRef):
1294  """
1295  Load FGCM parameters from a previous fit cycle
1296 
1297  Parameters
1298  ----------
1299  parDataRef : `lsst.daf.persistence.ButlerDataRef` or `lsst.daf.butler.DeferredDatasetHandle`
1300  dataRef for previous fit parameter catalog.
1301 
1302  Returns
1303  -------
1304  inParInfo: `numpy.ndarray`
1305  Numpy array parameter information formatted for input to fgcm
1306  inParameters: `numpy.ndarray`
1307  Numpy array parameter values formatted for input to fgcm
1308  inSuperStar: `numpy.array`
1309  Superstar flat formatted for input to fgcm
1310  """
1311  # note that we already checked that this is available
1312  parCat = parDataRef.get()
1313 
1314  parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1315  parFitBands = np.array(parCat[0]['fitBands'].split(','))
1316 
1317  inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1318  ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1319  (parLutFilterNames.size, )),
1320  ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1321  ('LNTAUUNIT', 'f8'),
1322  ('LNTAUSLOPEUNIT', 'f8'),
1323  ('ALPHAUNIT', 'f8'),
1324  ('LNPWVUNIT', 'f8'),
1325  ('LNPWVSLOPEUNIT', 'f8'),
1326  ('LNPWVQUADRATICUNIT', 'f8'),
1327  ('LNPWVGLOBALUNIT', 'f8'),
1328  ('O3UNIT', 'f8'),
1329  ('QESYSUNIT', 'f8'),
1330  ('FILTEROFFSETUNIT', 'f8'),
1331  ('HASEXTERNALPWV', 'i2'),
1332  ('HASEXTERNALTAU', 'i2')])
1333  inParInfo['NCCD'] = parCat['nCcd']
1334  inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1335  inParInfo['FITBANDS'][:] = parFitBands
1336  inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1337  inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1338 
1339  inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1340  ('PARO3', 'f8', (parCat['parO3'].size, )),
1341  ('PARLNTAUINTERCEPT', 'f8',
1342  (parCat['parLnTauIntercept'].size, )),
1343  ('PARLNTAUSLOPE', 'f8',
1344  (parCat['parLnTauSlope'].size, )),
1345  ('PARLNPWVINTERCEPT', 'f8',
1346  (parCat['parLnPwvIntercept'].size, )),
1347  ('PARLNPWVSLOPE', 'f8',
1348  (parCat['parLnPwvSlope'].size, )),
1349  ('PARLNPWVQUADRATIC', 'f8',
1350  (parCat['parLnPwvQuadratic'].size, )),
1351  ('PARQESYSINTERCEPT', 'f8',
1352  (parCat['parQeSysIntercept'].size, )),
1353  ('COMPQESYSSLOPE', 'f8',
1354  (parCat['compQeSysSlope'].size, )),
1355  ('PARFILTEROFFSET', 'f8',
1356  (parCat['parFilterOffset'].size, )),
1357  ('PARFILTEROFFSETFITFLAG', 'i2',
1358  (parCat['parFilterOffsetFitFlag'].size, )),
1359  ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1360  ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1361  ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1362  (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1363  ('COMPABSTHROUGHPUT', 'f8',
1364  (parCat['compAbsThroughput'].size, )),
1365  ('COMPREFOFFSET', 'f8',
1366  (parCat['compRefOffset'].size, )),
1367  ('COMPREFSIGMA', 'f8',
1368  (parCat['compRefSigma'].size, )),
1369  ('COMPMIRRORCHROMATICITY', 'f8',
1370  (parCat['compMirrorChromaticity'].size, )),
1371  ('MIRRORCHROMATICITYPIVOT', 'f8',
1372  (parCat['mirrorChromaticityPivot'].size, )),
1373  ('COMPMEDIANSEDSLOPE', 'f8',
1374  (parCat['compMedianSedSlope'].size, )),
1375  ('COMPAPERCORRPIVOT', 'f8',
1376  (parCat['compAperCorrPivot'].size, )),
1377  ('COMPAPERCORRSLOPE', 'f8',
1378  (parCat['compAperCorrSlope'].size, )),
1379  ('COMPAPERCORRSLOPEERR', 'f8',
1380  (parCat['compAperCorrSlopeErr'].size, )),
1381  ('COMPAPERCORRRANGE', 'f8',
1382  (parCat['compAperCorrRange'].size, )),
1383  ('COMPMODELERREXPTIMEPIVOT', 'f8',
1384  (parCat['compModelErrExptimePivot'].size, )),
1385  ('COMPMODELERRFWHMPIVOT', 'f8',
1386  (parCat['compModelErrFwhmPivot'].size, )),
1387  ('COMPMODELERRSKYPIVOT', 'f8',
1388  (parCat['compModelErrSkyPivot'].size, )),
1389  ('COMPMODELERRPARS', 'f8',
1390  (parCat['compModelErrPars'].size, )),
1391  ('COMPEXPGRAY', 'f8',
1392  (parCat['compExpGray'].size, )),
1393  ('COMPVARGRAY', 'f8',
1394  (parCat['compVarGray'].size, )),
1395  ('COMPEXPDELTAMAGBKG', 'f8',
1396  (parCat['compExpDeltaMagBkg'].size, )),
1397  ('COMPNGOODSTARPEREXP', 'i4',
1398  (parCat['compNGoodStarPerExp'].size, )),
1399  ('COMPSIGFGCM', 'f8',
1400  (parCat['compSigFgcm'].size, )),
1401  ('COMPSIGMACAL', 'f8',
1402  (parCat['compSigmaCal'].size, )),
1403  ('COMPRETRIEVEDLNPWV', 'f8',
1404  (parCat['compRetrievedLnPwv'].size, )),
1405  ('COMPRETRIEVEDLNPWVRAW', 'f8',
1406  (parCat['compRetrievedLnPwvRaw'].size, )),
1407  ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1408  (parCat['compRetrievedLnPwvFlag'].size, )),
1409  ('COMPRETRIEVEDTAUNIGHT', 'f8',
1410  (parCat['compRetrievedTauNight'].size, ))])
1411 
1412  inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1413  inParams['PARO3'][:] = parCat['parO3'][0, :]
1414  inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1415  inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1416  inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1417  inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1418  inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1419  inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1420  inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1421  inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1422  inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1423  inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1424  inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1425  inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1426  inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1427  inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1428  inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1429  inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1430  inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1431  inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1432  inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1433  inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1434  inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1435  inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1436  inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1437  inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1438  inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1439  inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1440  inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1441  inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1442  inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1443  inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1444  inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1445  inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1446  inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1447  inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1448  inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1449  inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1450 
1451  inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1452  inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1453 
1454  return (inParInfo, inParams, inSuperStar)
1455 
1456  def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1457  """
1458  Persist FGCM datasets through the butler.
1459 
1460  Parameters
1461  ----------
1462  fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1463  Fgcm Fit cycle object
1464  """
1465  fgcmDatasetDict = {}
1466 
1467  # Save the parameters
1468  parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1469 
1470  parSchema = afwTable.Schema()
1471 
1472  comma = ','
1473  lutFilterNameString = comma.join([n.decode('utf-8')
1474  for n in parInfo['LUTFILTERNAMES'][0]])
1475  fitBandString = comma.join([n.decode('utf-8')
1476  for n in parInfo['FITBANDS'][0]])
1477 
1478  parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1479  lutFilterNameString, fitBandString)
1480  parCat = self._makeParCatalog(parSchema, parInfo, pars,
1481  fgcmFitCycle.fgcmPars.parSuperStarFlat,
1482  lutFilterNameString, fitBandString)
1483 
1484  fgcmDatasetDict['fgcmFitParameters'] = parCat
1485 
1486  # Save the indices of the flagged stars
1487  # (stars that have been (a) reserved from the fit for testing and
1488  # (b) bad stars that have failed quality checks.)
1489  flagStarSchema = self._makeFlagStarSchema()
1490  flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1491  flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1492 
1493  fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1494 
1495  # Save the zeropoint information and atmospheres only if desired
1496  if self.outputZeropoints:
1497  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1498  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1499 
1500  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1501  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1502 
1503  fgcmDatasetDict['fgcmZeropoints'] = zptCat
1504 
1505  # Save atmosphere values
1506  # These are generated by the same code that generates zeropoints
1507  atmSchema = makeAtmSchema()
1508  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1509 
1510  fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1511 
1512  # Save the standard stars (if configured)
1513  if self.outputStandards:
1514  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1515  stdSchema = makeStdSchema(len(goodBands))
1516  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1517 
1518  fgcmDatasetDict['fgcmStandardStars'] = stdCat
1519 
1520  return fgcmDatasetDict
1521 
1522  def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1523  lutFilterNameString, fitBandString):
1524  """
1525  Make the parameter persistence schema
1526 
1527  Parameters
1528  ----------
1529  parInfo: `numpy.ndarray`
1530  Parameter information returned by fgcm
1531  pars: `numpy.ndarray`
1532  Parameter values returned by fgcm
1533  parSuperStarFlat: `numpy.array`
1534  Superstar flat values returned by fgcm
1535  lutFilterNameString: `str`
1536  Combined string of all the lutFilterNames
1537  fitBandString: `str`
1538  Combined string of all the fitBands
1539 
1540  Returns
1541  -------
1542  parSchema: `afwTable.schema`
1543  """
1544 
1545  parSchema = afwTable.Schema()
1546 
1547  # parameter info section
1548  parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1549  parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1550  size=len(lutFilterNameString))
1551  parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1552  size=len(fitBandString))
1553  parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1554  parSchema.addField('lnTauSlopeUnit', type=np.float64,
1555  doc='Step units for ln(AOD) slope')
1556  parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1557  parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1558  parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1559  doc='Step units for ln(pwv) slope')
1560  parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1561  doc='Step units for ln(pwv) quadratic term')
1562  parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1563  doc='Step units for global ln(pwv) parameters')
1564  parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1565  parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1566  parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1567  parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1568  parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1569 
1570  # parameter section
1571  parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1572  size=pars['PARALPHA'].size)
1573  parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1574  size=pars['PARO3'].size)
1575  parSchema.addField('parLnTauIntercept', type='ArrayD',
1576  doc='ln(Tau) intercept parameter vector',
1577  size=pars['PARLNTAUINTERCEPT'].size)
1578  parSchema.addField('parLnTauSlope', type='ArrayD',
1579  doc='ln(Tau) slope parameter vector',
1580  size=pars['PARLNTAUSLOPE'].size)
1581  parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1582  size=pars['PARLNPWVINTERCEPT'].size)
1583  parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1584  size=pars['PARLNPWVSLOPE'].size)
1585  parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1586  size=pars['PARLNPWVQUADRATIC'].size)
1587  parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1588  size=pars['PARQESYSINTERCEPT'].size)
1589  parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1590  size=pars[0]['COMPQESYSSLOPE'].size)
1591  parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1592  size=pars['PARFILTEROFFSET'].size)
1593  parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1594  size=pars['PARFILTEROFFSETFITFLAG'].size)
1595  parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1596  doc='Global scale for retrieved ln(pwv)')
1597  parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1598  doc='Global offset for retrieved ln(pwv)')
1599  parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1600  doc='Nightly offset for retrieved ln(pwv)',
1601  size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1602  parSchema.addField('compAbsThroughput', type='ArrayD',
1603  doc='Absolute throughput (relative to transmission curves)',
1604  size=pars['COMPABSTHROUGHPUT'].size)
1605  parSchema.addField('compRefOffset', type='ArrayD',
1606  doc='Offset between reference stars and calibrated stars',
1607  size=pars['COMPREFOFFSET'].size)
1608  parSchema.addField('compRefSigma', type='ArrayD',
1609  doc='Width of reference star/calibrated star distribution',
1610  size=pars['COMPREFSIGMA'].size)
1611  parSchema.addField('compMirrorChromaticity', type='ArrayD',
1612  doc='Computed mirror chromaticity terms',
1613  size=pars['COMPMIRRORCHROMATICITY'].size)
1614  parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1615  doc='Mirror chromaticity pivot mjd',
1616  size=pars['MIRRORCHROMATICITYPIVOT'].size)
1617  parSchema.addField('compMedianSedSlope', type='ArrayD',
1618  doc='Computed median SED slope (per band)',
1619  size=pars['COMPMEDIANSEDSLOPE'].size)
1620  parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1621  size=pars['COMPAPERCORRPIVOT'].size)
1622  parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1623  size=pars['COMPAPERCORRSLOPE'].size)
1624  parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1625  size=pars['COMPAPERCORRSLOPEERR'].size)
1626  parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1627  size=pars['COMPAPERCORRRANGE'].size)
1628  parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1629  size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1630  parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1631  size=pars['COMPMODELERRFWHMPIVOT'].size)
1632  parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1633  size=pars['COMPMODELERRSKYPIVOT'].size)
1634  parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1635  size=pars['COMPMODELERRPARS'].size)
1636  parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1637  size=pars['COMPEXPGRAY'].size)
1638  parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1639  size=pars['COMPVARGRAY'].size)
1640  parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1641  doc='Computed exposure offset due to background',
1642  size=pars['COMPEXPDELTAMAGBKG'].size)
1643  parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1644  doc='Computed number of good stars per exposure',
1645  size=pars['COMPNGOODSTARPEREXP'].size)
1646  parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1647  size=pars['COMPSIGFGCM'].size)
1648  parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1649  size=pars['COMPSIGMACAL'].size)
1650  parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1651  size=pars['COMPRETRIEVEDLNPWV'].size)
1652  parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1653  size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1654  parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1655  size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1656  parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1657  size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1658  # superstarflat section
1659  parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1660  size=4)
1661  parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1662  size=parSuperStarFlat.size)
1663 
1664  return parSchema
1665 
1666  def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1667  lutFilterNameString, fitBandString):
1668  """
1669  Make the FGCM parameter catalog for persistence
1670 
1671  Parameters
1672  ----------
1673  parSchema: `lsst.afw.table.Schema`
1674  Parameter catalog schema
1675  pars: `numpy.ndarray`
1676  FGCM parameters to put into parCat
1677  parSuperStarFlat: `numpy.array`
1678  FGCM superstar flat array to put into parCat
1679  lutFilterNameString: `str`
1680  Combined string of all the lutFilterNames
1681  fitBandString: `str`
1682  Combined string of all the fitBands
1683 
1684  Returns
1685  -------
1686  parCat: `afwTable.BasicCatalog`
1687  Atmosphere and instrumental model parameter catalog for persistence
1688  """
1689 
1690  parCat = afwTable.BaseCatalog(parSchema)
1691  parCat.reserve(1)
1692 
1693  # The parameter catalog just has one row, with many columns for all the
1694  # atmosphere and instrument fit parameters
1695  rec = parCat.addNew()
1696 
1697  # info section
1698  rec['nCcd'] = parInfo['NCCD']
1699  rec['lutFilterNames'] = lutFilterNameString
1700  rec['fitBands'] = fitBandString
1701  # note these are not currently supported here.
1702  rec['hasExternalPwv'] = 0
1703  rec['hasExternalTau'] = 0
1704 
1705  # parameter section
1706 
1707  scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1708 
1709  arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1710  'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1711  'parQeSysIntercept', 'compQeSysSlope',
1712  'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1713  'parFilterOffset', 'parFilterOffsetFitFlag',
1714  'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1715  'compMirrorChromaticity', 'mirrorChromaticityPivot',
1716  'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1717  'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1718  'compModelErrSkyPivot', 'compModelErrPars',
1719  'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1720  'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1721  'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1722  'compRetrievedTauNight']
1723 
1724  for scalarName in scalarNames:
1725  rec[scalarName] = pars[scalarName.upper()]
1726 
1727  for arrName in arrNames:
1728  rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1729 
1730  # superstar section
1731  rec['superstarSize'][:] = parSuperStarFlat.shape
1732  rec['superstar'][:] = parSuperStarFlat.flatten()
1733 
1734  return parCat
1735 
1736  def _makeFlagStarSchema(self):
1737  """
1738  Make the flagged-stars schema
1739 
1740  Returns
1741  -------
1742  flagStarSchema: `lsst.afw.table.Schema`
1743  """
1744 
1745  flagStarSchema = afwTable.Schema()
1746 
1747  flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1748  flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1749 
1750  return flagStarSchema
1751 
1752  def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1753  """
1754  Make the flagged star catalog for persistence
1755 
1756  Parameters
1757  ----------
1758  flagStarSchema: `lsst.afw.table.Schema`
1759  Flagged star schema
1760  flagStarStruct: `numpy.ndarray`
1761  Flagged star structure from fgcm
1762 
1763  Returns
1764  -------
1765  flagStarCat: `lsst.afw.table.BaseCatalog`
1766  Flagged star catalog for persistence
1767  """
1768 
1769  flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1770  flagStarCat.resize(flagStarStruct.size)
1771 
1772  flagStarCat['objId'][:] = flagStarStruct['OBJID']
1773  flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1774 
1775  return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:874
def makeStdSchema(nBands)
Definition: utilities.py:729
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
Definition: utilities.py:49
def translateFgcmLut(lutCat, filterMap)
Definition: utilities.py:208
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:696
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:617
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:761
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:531
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:390
def translateVisitCatalog(visitCat)
Definition: utilities.py:339