lsst.fgcmcal  21.0.0-15-g543e7c3+54360a7183
fgcmFitCycle.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Perform a single fit cycle of FGCM.
24 
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
28 for the global fit).
29 
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
34 """
35 
36 import sys
37 import traceback
38 import copy
39 
40 import numpy as np
41 
42 import lsst.pex.config as pexConfig
43 import lsst.pipe.base as pipeBase
44 from lsst.pipe.base import connectionTypes
45 import lsst.afw.table as afwTable
46 
47 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
48 from .utilities import extractReferenceMags
49 from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
50 from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51 from .sedterms import SedboundarytermDict, SedtermDict
52 from .utilities import lookupStaticCalibrations
53 
54 import fgcm
55 
56 __all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57 
58 MULTIPLE_CYCLES_MAX = 10
59 
60 
61 class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
62  dimensions=("instrument",),
63  defaultTemplates={"previousCycleNumber": "-1",
64  "cycleNumber": "0"}):
65  camera = connectionTypes.PrerequisiteInput(
66  doc="Camera instrument",
67  name="camera",
68  storageClass="Camera",
69  dimensions=("instrument",),
70  lookupFunction=lookupStaticCalibrations,
71  isCalibration=True,
72  )
73 
74  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
75  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
76  "chromatic corrections."),
77  name="fgcmLookUpTable",
78  storageClass="Catalog",
79  dimensions=("instrument",),
80  deferLoad=True,
81  )
82 
83  fgcmVisitCatalog = connectionTypes.Input(
84  doc="Catalog of visit information for fgcm",
85  name="fgcmVisitCatalog",
86  storageClass="Catalog",
87  dimensions=("instrument",),
88  deferLoad=True,
89  )
90 
91  fgcmStarObservations = connectionTypes.Input(
92  doc="Catalog of star observations for fgcm",
93  name="fgcmStarObservations",
94  storageClass="Catalog",
95  dimensions=("instrument",),
96  deferLoad=True,
97  )
98 
99  fgcmStarIds = connectionTypes.Input(
100  doc="Catalog of fgcm calibration star IDs",
101  name="fgcmStarIds",
102  storageClass="Catalog",
103  dimensions=("instrument",),
104  deferLoad=True,
105  )
106 
107  fgcmStarIndices = connectionTypes.Input(
108  doc="Catalog of fgcm calibration star indices",
109  name="fgcmStarIndices",
110  storageClass="Catalog",
111  dimensions=("instrument",),
112  deferLoad=True,
113  )
114 
115  fgcmReferenceStars = connectionTypes.Input(
116  doc="Catalog of fgcm-matched reference stars",
117  name="fgcmReferenceStars",
118  storageClass="Catalog",
119  dimensions=("instrument",),
120  deferLoad=True,
121  )
122 
123  fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
124  doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
125  name="fgcmFlaggedStars{previousCycleNumber}",
126  storageClass="Catalog",
127  dimensions=("instrument",),
128  deferLoad=True,
129  )
130 
131  fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
132  doc="Catalog of fgcm fit parameters from previous fit cycle",
133  name="fgcmFitParameters{previousCycleNumber}",
134  storageClass="Catalog",
135  dimensions=("instrument",),
136  deferLoad=True,
137  )
138 
139  fgcmFitParameters = connectionTypes.Output(
140  doc="Catalog of fgcm fit parameters from current fit cycle",
141  name="fgcmFitParameters{cycleNumber}",
142  storageClass="Catalog",
143  dimensions=("instrument",),
144  )
145 
146  fgcmFlaggedStars = connectionTypes.Output(
147  doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
148  name="fgcmFlaggedStars{cycleNumber}",
149  storageClass="Catalog",
150  dimensions=("instrument",),
151  )
152 
153  fgcmZeropoints = connectionTypes.Output(
154  doc="Catalog of fgcm zeropoint data from current fit cycle",
155  name="fgcmZeropoints{cycleNumber}",
156  storageClass="Catalog",
157  dimensions=("instrument",),
158  )
159 
160  fgcmAtmosphereParameters = connectionTypes.Output(
161  doc="Catalog of atmospheric fit parameters from current fit cycle",
162  name="fgcmAtmosphereParameters{cycleNumber}",
163  storageClass="Catalog",
164  dimensions=("instrument",),
165  )
166 
167  fgcmStandardStars = connectionTypes.Output(
168  doc="Catalog of standard star magnitudes from current fit cycle",
169  name="fgcmStandardStars{cycleNumber}",
170  storageClass="SimpleCatalog",
171  dimensions=("instrument",),
172  )
173 
174  # Add connections for running multiple cycles
175  # This uses vars() to alter the class __dict__ to programmatically
176  # write many similar outputs.
177  for cycle in range(MULTIPLE_CYCLES_MAX):
178  vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
179  doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
180  name=f"fgcmFitParameters{cycle}",
181  storageClass="Catalog",
182  dimensions=("instrument",),
183  )
184  vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
185  doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
186  name=f"fgcmFlaggedStars{cycle}",
187  storageClass="Catalog",
188  dimensions=("instrument",),
189  )
190  vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
191  doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
192  name=f"fgcmZeropoints{cycle}",
193  storageClass="Catalog",
194  dimensions=("instrument",),
195  )
196  vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
197  doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
198  name=f"fgcmAtmosphereParameters{cycle}",
199  storageClass="Catalog",
200  dimensions=("instrument",),
201  )
202  vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
203  doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
204  name=f"fgcmStandardStars{cycle}",
205  storageClass="SimpleCatalog",
206  dimensions=("instrument",),
207  )
208 
209  def __init__(self, *, config=None):
210  super().__init__(config=config)
211 
212  if not config.doReferenceCalibration:
213  self.inputs.remove("fgcmReferenceStars")
214 
215  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
216  raise ValueError("cycleNumber must be of integer format")
217  if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
218  raise ValueError("previousCycleNumber must be of integer format")
219  if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
220  raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
221 
222  if int(config.connections.cycleNumber) == 0:
223  self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
224  self.prerequisiteInputs.remove("fgcmFitParametersInput")
225 
226  if not self.config.doMultipleCycles:
227  # Single-cycle run
228  if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
229  self.outputs.remove("fgcmStandardStars")
230 
231  if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
232  self.outputs.remove("fgcmZeropoints")
233  self.outputs.remove("fgcmAtmosphereParameters")
234 
235  # Remove all multiple cycle outputs
236  for cycle in range(0, MULTIPLE_CYCLES_MAX):
237  self.outputs.remove(f"fgcmFitParameters{cycle}")
238  self.outputs.remove(f"fgcmFlaggedStars{cycle}")
239  self.outputs.remove(f"fgcmZeropoints{cycle}")
240  self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
241  self.outputs.remove(f"fgcmStandardStars{cycle}")
242 
243  else:
244  # Multiple-cycle run
245  # Remove single-cycle outputs
246  self.outputs.remove("fgcmFitParameters")
247  self.outputs.remove("fgcmFlaggedStars")
248  self.outputs.remove("fgcmZeropoints")
249  self.outputs.remove("fgcmAtmosphereParameters")
250  self.outputs.remove("fgcmStandardStars")
251 
252  # Remove outputs from cycles that are not used
253  for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
254  MULTIPLE_CYCLES_MAX):
255  self.outputs.remove(f"fgcmFitParameters{cycle}")
256  self.outputs.remove(f"fgcmFlaggedStars{cycle}")
257  self.outputs.remove(f"fgcmZeropoints{cycle}")
258  self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
259  self.outputs.remove(f"fgcmStandardStars{cycle}")
260 
261  # Remove non-final-cycle outputs if necessary
262  for cycle in range(self.config.multipleCyclesFinalCycleNumber):
263  if not self.config.outputZeropointsBeforeFinalCycle:
264  self.outputs.remove(f"fgcmZeropoints{cycle}")
265  self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
266  if not self.config.outputStandardsBeforeFinalCycle:
267  self.outputs.remove(f"fgcmStandardStars{cycle}")
268 
269 
270 class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
271  pipelineConnections=FgcmFitCycleConnections):
272  """Config for FgcmFitCycle"""
273 
274  doMultipleCycles = pexConfig.Field(
275  doc="Run multiple fit cycles in one task",
276  dtype=bool,
277  default=False,
278  )
279  multipleCyclesFinalCycleNumber = pexConfig.RangeField(
280  doc=("Final cycle number in multiple cycle mode. The initial cycle "
281  "is 0, with limited parameters fit. The next cycle is 1 with "
282  "full parameter fit. The final cycle is a clean-up with no "
283  "parameters fit. There will be a total of "
284  "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
285  "cycle number cannot be less than 2."),
286  dtype=int,
287  default=5,
288  min=2,
289  max=MULTIPLE_CYCLES_MAX,
290  inclusiveMax=True,
291  )
292  bands = pexConfig.ListField(
293  doc="Bands to run calibration",
294  dtype=str,
295  default=[],
296  )
297  fitFlag = pexConfig.ListField(
298  doc=("Flag for which bands are directly constrained in the FGCM fit. "
299  "Bands set to 0 will have the atmosphere constrained from observations "
300  "in other bands on the same night. Must be same length as config.bands, "
301  "and matched band-by-band."),
302  dtype=int,
303  default=(0,),
304  optional=True,
305  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
306  "It will be removed after v20. Use fitBands instead."),
307  )
308  fitBands = pexConfig.ListField(
309  doc=("Bands to use in atmospheric fit. The bands not listed here will have "
310  "the atmosphere constrained from the 'fitBands' on the same night. "
311  "Must be a subset of `config.bands`"),
312  dtype=str,
313  default=[],
314  )
315  requiredFlag = pexConfig.ListField(
316  doc=("Flag for which bands are required for a star to be considered a calibration "
317  "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
318  "be same length as config.bands, and matched band-by-band."),
319  dtype=int,
320  default=(0,),
321  optional=True,
322  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
323  "It will be removed after v20. Use requiredBands instead."),
324  )
325  requiredBands = pexConfig.ListField(
326  doc=("Bands that are required for a star to be considered a calibration star. "
327  "Must be a subset of `config.bands`"),
328  dtype=str,
329  default=[],
330  )
331  filterMap = pexConfig.DictField(
332  doc="Mapping from 'filterName' to band.",
333  keytype=str,
334  itemtype=str,
335  default={},
336  deprecated=("This field is no longer used, and has been deprecated by "
337  "DM-28088. It will be removed after v22. Use "
338  "physicalFilterMap instead.")
339  )
340  # The following config will not be necessary after Gen2 retirement.
341  # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
342  # is easiest to access in the config file.
343  physicalFilterMap = pexConfig.DictField(
344  doc="Mapping from 'physicalFilter' to band.",
345  keytype=str,
346  itemtype=str,
347  default={},
348  )
349  doReferenceCalibration = pexConfig.Field(
350  doc="Use reference catalog as additional constraint on calibration",
351  dtype=bool,
352  default=True,
353  )
354  refStarSnMin = pexConfig.Field(
355  doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
356  dtype=float,
357  default=50.0,
358  )
359  refStarOutlierNSig = pexConfig.Field(
360  doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
361  "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
362  dtype=float,
363  default=4.0,
364  )
365  applyRefStarColorCuts = pexConfig.Field(
366  doc="Apply color cuts to reference stars?",
367  dtype=bool,
368  default=True,
369  )
370  nCore = pexConfig.Field(
371  doc="Number of cores to use",
372  dtype=int,
373  default=4,
374  )
375  nStarPerRun = pexConfig.Field(
376  doc="Number of stars to run in each chunk",
377  dtype=int,
378  default=200000,
379  )
380  nExpPerRun = pexConfig.Field(
381  doc="Number of exposures to run in each chunk",
382  dtype=int,
383  default=1000,
384  )
385  reserveFraction = pexConfig.Field(
386  doc="Fraction of stars to reserve for testing",
387  dtype=float,
388  default=0.1,
389  )
390  freezeStdAtmosphere = pexConfig.Field(
391  doc="Freeze atmosphere parameters to standard (for testing)",
392  dtype=bool,
393  default=False,
394  )
395  precomputeSuperStarInitialCycle = pexConfig.Field(
396  doc="Precompute superstar flat for initial cycle",
397  dtype=bool,
398  default=False,
399  )
400  superStarSubCcd = pexConfig.Field(
401  doc="Compute superstar flat on sub-ccd scale",
402  dtype=bool,
403  default=True,
404  optional=True,
405  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
406  "It will be removed after v20. Use superStarSubCcdDict instead."),
407  )
408  superStarSubCcdDict = pexConfig.DictField(
409  doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
410  "Must have one entry per band."),
411  keytype=str,
412  itemtype=bool,
413  default={},
414  )
415  superStarSubCcdChebyshevOrder = pexConfig.Field(
416  doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
417  "Global default is first-order polynomials, and should be overridden "
418  "on a camera-by-camera basis depending on the ISR."),
419  dtype=int,
420  default=1,
421  )
422  superStarSubCcdTriangular = pexConfig.Field(
423  doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
424  "suppress high-order cross terms?"),
425  dtype=bool,
426  default=False,
427  )
428  superStarSigmaClip = pexConfig.Field(
429  doc="Number of sigma to clip outliers when selecting for superstar flats",
430  dtype=float,
431  default=5.0,
432  )
433  focalPlaneSigmaClip = pexConfig.Field(
434  doc="Number of sigma to clip outliers per focal-plane.",
435  dtype=float,
436  default=4.0,
437  )
438  ccdGraySubCcd = pexConfig.Field(
439  doc="Compute CCD gray terms on sub-ccd scale",
440  dtype=bool,
441  default=False,
442  optional=True,
443  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
444  "It will be removed after v20. Use ccdGraySubCcdDict instead."),
445  )
446  ccdGraySubCcdDict = pexConfig.DictField(
447  doc=("Per-band specification on whether to compute achromatic per-ccd residual "
448  "('ccd gray') on a sub-ccd scale."),
449  keytype=str,
450  itemtype=bool,
451  default={},
452  )
453  ccdGraySubCcdChebyshevOrder = pexConfig.Field(
454  doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
455  dtype=int,
456  default=1,
457  )
458  ccdGraySubCcdTriangular = pexConfig.Field(
459  doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
460  "suppress high-order cross terms?"),
461  dtype=bool,
462  default=True,
463  )
464  ccdGrayFocalPlaneDict = pexConfig.DictField(
465  doc=("Per-band specification on whether to compute focal-plane residual "
466  "('ccd gray') corrections."),
467  keytype=str,
468  itemtype=bool,
469  default={},
470  )
471  ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
472  doc=("Minimum number of 'good' CCDs required to perform focal-plane "
473  "gray corrections. If there are fewer good CCDs then the gray "
474  "correction is computed per-ccd."),
475  dtype=int,
476  default=1,
477  )
478  ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
479  doc="Order of the 2D chebyshev polynomials for focal plane fit.",
480  dtype=int,
481  default=3,
482  )
483  cycleNumber = pexConfig.Field(
484  doc=("FGCM fit cycle number. This is automatically incremented after each run "
485  "and stage of outlier rejection. See cookbook for details."),
486  dtype=int,
487  default=None,
488  )
489  isFinalCycle = pexConfig.Field(
490  doc=("Is this the final cycle of the fitting? Will automatically compute final "
491  "selection of stars and photometric exposures, and will output zeropoints "
492  "and standard stars for use in fgcmOutputProducts"),
493  dtype=bool,
494  default=False,
495  )
496  maxIterBeforeFinalCycle = pexConfig.Field(
497  doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
498  "will always be 0 in the final cycle for cleanup and final selection."),
499  dtype=int,
500  default=50,
501  )
502  deltaMagBkgOffsetPercentile = pexConfig.Field(
503  doc=("Percentile brightest stars on a visit/ccd to use to compute net "
504  "offset from local background subtraction."),
505  dtype=float,
506  default=0.25,
507  )
508  deltaMagBkgPerCcd = pexConfig.Field(
509  doc=("Compute net offset from local background subtraction per-ccd? "
510  "Otherwise, use computation per visit."),
511  dtype=bool,
512  default=False,
513  )
514  utBoundary = pexConfig.Field(
515  doc="Boundary (in UTC) from day-to-day",
516  dtype=float,
517  default=None,
518  )
519  washMjds = pexConfig.ListField(
520  doc="Mirror wash MJDs",
521  dtype=float,
522  default=(0.0,),
523  )
524  epochMjds = pexConfig.ListField(
525  doc="Epoch boundaries in MJD",
526  dtype=float,
527  default=(0.0,),
528  )
529  minObsPerBand = pexConfig.Field(
530  doc="Minimum good observations per band",
531  dtype=int,
532  default=2,
533  )
534  # TODO: When DM-16511 is done, it will be possible to get the
535  # telescope latitude directly from the camera.
536  latitude = pexConfig.Field(
537  doc="Observatory latitude",
538  dtype=float,
539  default=None,
540  )
541  brightObsGrayMax = pexConfig.Field(
542  doc="Maximum gray extinction to be considered bright observation",
543  dtype=float,
544  default=0.15,
545  )
546  minStarPerCcd = pexConfig.Field(
547  doc=("Minimum number of good stars per CCD to be used in calibration fit. "
548  "CCDs with fewer stars will have their calibration estimated from other "
549  "CCDs in the same visit, with zeropoint error increased accordingly."),
550  dtype=int,
551  default=5,
552  )
553  minCcdPerExp = pexConfig.Field(
554  doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
555  "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
556  dtype=int,
557  default=5,
558  )
559  maxCcdGrayErr = pexConfig.Field(
560  doc="Maximum error on CCD gray offset to be considered photometric",
561  dtype=float,
562  default=0.05,
563  )
564  minStarPerExp = pexConfig.Field(
565  doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
566  "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
567  dtype=int,
568  default=600,
569  )
570  minExpPerNight = pexConfig.Field(
571  doc="Minimum number of good exposures/visits to consider a partly photometric night",
572  dtype=int,
573  default=10,
574  )
575  expGrayInitialCut = pexConfig.Field(
576  doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
577  "observations."),
578  dtype=float,
579  default=-0.25,
580  )
581  expGrayPhotometricCut = pexConfig.ListField(
582  doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
583  "Must be same length as config.bands, and matched band-by-band."),
584  dtype=float,
585  default=(0.0,),
586  optional=True,
587  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
588  "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
589  )
590  expGrayPhotometricCutDict = pexConfig.DictField(
591  doc=("Per-band specification on maximum (negative) achromatic exposure residual "
592  "('gray term') for a visit to be considered photometric. Must have one "
593  "entry per band. Broad-band filters should be -0.05."),
594  keytype=str,
595  itemtype=float,
596  default={},
597  )
598  expGrayHighCut = pexConfig.ListField(
599  doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
600  "Must be same length as config.bands, and matched band-by-band."),
601  dtype=float,
602  default=(0.0,),
603  optional=True,
604  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
605  "It will be removed after v20. Use expGrayHighCutDict instead."),
606  )
607  expGrayHighCutDict = pexConfig.DictField(
608  doc=("Per-band specification on maximum (positive) achromatic exposure residual "
609  "('gray term') for a visit to be considered photometric. Must have one "
610  "entry per band. Broad-band filters should be 0.2."),
611  keytype=str,
612  itemtype=float,
613  default={},
614  )
615  expGrayRecoverCut = pexConfig.Field(
616  doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
617  "Visits with more gray extinction will only get CCD zeropoints if there are "
618  "sufficient star observations (minStarPerCcd) on that CCD."),
619  dtype=float,
620  default=-1.0,
621  )
622  expVarGrayPhotometricCut = pexConfig.Field(
623  doc="Maximum exposure variance to be considered possibly photometric",
624  dtype=float,
625  default=0.0005,
626  optional=True,
627  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
628  "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
629  )
630  expVarGrayPhotometricCutDict = pexConfig.DictField(
631  doc=("Per-band specification on maximum exposure variance to be considered possibly "
632  "photometric. Must have one entry per band. Broad-band filters should be "
633  "0.0005."),
634  keytype=str,
635  itemtype=float,
636  default={},
637  )
638  expGrayErrRecoverCut = pexConfig.Field(
639  doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
640  "Visits with more gray variance will only get CCD zeropoints if there are "
641  "sufficient star observations (minStarPerCcd) on that CCD."),
642  dtype=float,
643  default=0.05,
644  )
645  aperCorrFitNBins = pexConfig.Field(
646  doc=("Number of aperture bins used in aperture correction fit. When set to 0"
647  "no fit will be performed, and the config.aperCorrInputSlopes will be "
648  "used if available."),
649  dtype=int,
650  default=10,
651  )
652  aperCorrInputSlopes = pexConfig.ListField(
653  doc=("Aperture correction input slope parameters. These are used on the first "
654  "fit iteration, and aperture correction parameters will be updated from "
655  "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
656  "when there is insufficient data to fit the parameters (e.g. tract mode). "
657  "If set, must be same length as config.bands, and matched band-by-band."),
658  dtype=float,
659  default=[],
660  optional=True,
661  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
662  "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
663  )
664  aperCorrInputSlopeDict = pexConfig.DictField(
665  doc=("Per-band specification of aperture correction input slope parameters. These "
666  "are used on the first fit iteration, and aperture correction parameters will "
667  "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
668  "to set this when there is insufficient data to fit the parameters (e.g. "
669  "tract mode)."),
670  keytype=str,
671  itemtype=float,
672  default={},
673  )
674  sedFudgeFactors = pexConfig.ListField(
675  doc=("Fudge factors for computing linear SED from colors. Must be same length as "
676  "config.bands, and matched band-by-band."),
677  dtype=float,
678  default=(0,),
679  optional=True,
680  deprecated=("This field has been deprecated and will be removed after v20. "
681  "Please use sedSlopeTermMap and sedSlopeMap."),
682  )
683  sedboundaryterms = pexConfig.ConfigField(
684  doc="Mapping from bands to SED boundary term names used is sedterms.",
685  dtype=SedboundarytermDict,
686  )
687  sedterms = pexConfig.ConfigField(
688  doc="Mapping from terms to bands for fgcm linear SED approximations.",
689  dtype=SedtermDict,
690  )
691  sigFgcmMaxErr = pexConfig.Field(
692  doc="Maximum mag error for fitting sigma_FGCM",
693  dtype=float,
694  default=0.01,
695  )
696  sigFgcmMaxEGray = pexConfig.ListField(
697  doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
698  "May be 1 element (same for all bands) or the same length as config.bands."),
699  dtype=float,
700  default=(0.05,),
701  optional=True,
702  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
703  "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
704  )
705  sigFgcmMaxEGrayDict = pexConfig.DictField(
706  doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
707  "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
708  "should be 0.05."),
709  keytype=str,
710  itemtype=float,
711  default={},
712  )
713  ccdGrayMaxStarErr = pexConfig.Field(
714  doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
715  "computation"),
716  dtype=float,
717  default=0.10,
718  )
719  approxThroughput = pexConfig.ListField(
720  doc=("Approximate overall throughput at start of calibration observations. "
721  "May be 1 element (same for all bands) or the same length as config.bands."),
722  dtype=float,
723  default=(1.0, ),
724  optional=True,
725  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
726  "It will be removed after v20. Use approxThroughputDict instead."),
727  )
728  approxThroughputDict = pexConfig.DictField(
729  doc=("Per-band specification of the approximate overall throughput at the start of "
730  "calibration observations. Must have one entry per band. Typically should "
731  "be 1.0."),
732  keytype=str,
733  itemtype=float,
734  default={},
735  )
736  sigmaCalRange = pexConfig.ListField(
737  doc="Allowed range for systematic error floor estimation",
738  dtype=float,
739  default=(0.001, 0.003),
740  )
741  sigmaCalFitPercentile = pexConfig.ListField(
742  doc="Magnitude percentile range to fit systematic error floor",
743  dtype=float,
744  default=(0.05, 0.15),
745  )
746  sigmaCalPlotPercentile = pexConfig.ListField(
747  doc="Magnitude percentile range to plot systematic error floor",
748  dtype=float,
749  default=(0.05, 0.95),
750  )
751  sigma0Phot = pexConfig.Field(
752  doc="Systematic error floor for all zeropoints",
753  dtype=float,
754  default=0.003,
755  )
756  mapLongitudeRef = pexConfig.Field(
757  doc="Reference longitude for plotting maps",
758  dtype=float,
759  default=0.0,
760  )
761  mapNSide = pexConfig.Field(
762  doc="Healpix nside for plotting maps",
763  dtype=int,
764  default=256,
765  )
766  outfileBase = pexConfig.Field(
767  doc="Filename start for plot output files",
768  dtype=str,
769  default=None,
770  )
771  starColorCuts = pexConfig.ListField(
772  doc="Encoded star-color cuts (to be cleaned up)",
773  dtype=str,
774  default=("NO_DATA",),
775  )
776  colorSplitIndices = pexConfig.ListField(
777  doc="Band indices to use to split stars by color",
778  dtype=int,
779  default=None,
780  optional=True,
781  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
782  "It will be removed after v20. Use colorSplitBands instead."),
783  )
784  colorSplitBands = pexConfig.ListField(
785  doc="Band names to use to split stars by color. Must have 2 entries.",
786  dtype=str,
787  length=2,
788  default=('g', 'i'),
789  )
790  modelMagErrors = pexConfig.Field(
791  doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
792  dtype=bool,
793  default=True,
794  )
795  useQuadraticPwv = pexConfig.Field(
796  doc="Model PWV with a quadratic term for variation through the night?",
797  dtype=bool,
798  default=False,
799  )
800  instrumentParsPerBand = pexConfig.Field(
801  doc=("Model instrumental parameters per band? "
802  "Otherwise, instrumental parameters (QE changes with time) are "
803  "shared among all bands."),
804  dtype=bool,
805  default=False,
806  )
807  instrumentSlopeMinDeltaT = pexConfig.Field(
808  doc=("Minimum time change (in days) between observations to use in constraining "
809  "instrument slope."),
810  dtype=float,
811  default=20.0,
812  )
813  fitMirrorChromaticity = pexConfig.Field(
814  doc="Fit (intraband) mirror chromatic term?",
815  dtype=bool,
816  default=False,
817  )
818  coatingMjds = pexConfig.ListField(
819  doc="Mirror coating dates in MJD",
820  dtype=float,
821  default=(0.0,),
822  )
823  outputStandardsBeforeFinalCycle = pexConfig.Field(
824  doc="Output standard stars prior to final cycle? Used in debugging.",
825  dtype=bool,
826  default=False,
827  )
828  outputZeropointsBeforeFinalCycle = pexConfig.Field(
829  doc="Output standard stars prior to final cycle? Used in debugging.",
830  dtype=bool,
831  default=False,
832  )
833  useRepeatabilityForExpGrayCuts = pexConfig.ListField(
834  doc=("Use star repeatability (instead of exposures) for computing photometric "
835  "cuts? Recommended for tract mode or bands with few exposures. "
836  "May be 1 element (same for all bands) or the same length as config.bands."),
837  dtype=bool,
838  default=(False,),
839  optional=True,
840  deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
841  "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
842  )
843  useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
844  doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
845  "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
846  keytype=str,
847  itemtype=bool,
848  default={},
849  )
850  autoPhotometricCutNSig = pexConfig.Field(
851  doc=("Number of sigma for automatic computation of (low) photometric cut. "
852  "Cut is based on exposure gray width (per band), unless "
853  "useRepeatabilityForExpGrayCuts is set, in which case the star "
854  "repeatability is used (also per band)."),
855  dtype=float,
856  default=3.0,
857  )
858  autoHighCutNSig = pexConfig.Field(
859  doc=("Number of sigma for automatic computation of (high) outlier cut. "
860  "Cut is based on exposure gray width (per band), unless "
861  "useRepeatabilityForExpGrayCuts is set, in which case the star "
862  "repeatability is used (also per band)."),
863  dtype=float,
864  default=4.0,
865  )
866  quietMode = pexConfig.Field(
867  doc="Be less verbose with logging.",
868  dtype=bool,
869  default=False,
870  )
871  doPlots = pexConfig.Field(
872  doc="Make fgcm QA plots.",
873  dtype=bool,
874  default=True,
875  )
876  randomSeed = pexConfig.Field(
877  doc="Random seed for fgcm for consistency in tests.",
878  dtype=int,
879  default=None,
880  optional=True,
881  )
882 
883  def validate(self):
884  super().validate()
885 
886  if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
887  msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
888  raise RuntimeError(msg)
889  if self.connections.cycleNumber != str(self.cycleNumber):
890  msg = "cycleNumber in template must be equal to connections.cycleNumber"
891  raise RuntimeError(msg)
892 
893  for band in self.fitBands:
894  if band not in self.bands:
895  msg = 'fitBand %s not in bands' % (band)
896  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
897  for band in self.requiredBands:
898  if band not in self.bands:
899  msg = 'requiredBand %s not in bands' % (band)
900  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
901  for band in self.colorSplitBands:
902  if band not in self.bands:
903  msg = 'colorSplitBand %s not in bands' % (band)
904  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
905  for band in self.bands:
906  if band not in self.superStarSubCcdDict:
907  msg = 'band %s not in superStarSubCcdDict' % (band)
908  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
909  self, msg)
910  if band not in self.ccdGraySubCcdDict:
911  msg = 'band %s not in ccdGraySubCcdDict' % (band)
912  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
913  self, msg)
914  if band not in self.expGrayPhotometricCutDict:
915  msg = 'band %s not in expGrayPhotometricCutDict' % (band)
916  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
917  self, msg)
918  if band not in self.expGrayHighCutDict:
919  msg = 'band %s not in expGrayHighCutDict' % (band)
920  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
921  self, msg)
922  if band not in self.expVarGrayPhotometricCutDict:
923  msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
924  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
925  self, msg)
926  if band not in self.sigFgcmMaxEGrayDict:
927  msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
928  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
929  self, msg)
930  if band not in self.approxThroughputDict:
931  msg = 'band %s not in approxThroughputDict' % (band)
932  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
933  self, msg)
934  if band not in self.useRepeatabilityForExpGrayCutsDict:
935  msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
936  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
937  self, msg)
938 
939 
940 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
941  """Subclass of TaskRunner for fgcmFitCycleTask
942 
943  fgcmFitCycleTask.run() takes one argument, the butler, and uses
944  stars and visits previously extracted from dataRefs by
945  fgcmBuildStars.
946  This Runner does not perform any dataRef parallelization, but the FGCM
947  code called by the Task uses python multiprocessing (see the "ncores"
948  config option).
949  """
950 
951  @staticmethod
952  def getTargetList(parsedCmd):
953  """
954  Return a list with one element, the butler.
955  """
956  return [parsedCmd.butler]
957 
958  def __call__(self, butler):
959  """
960  Parameters
961  ----------
962  butler: `lsst.daf.persistence.Butler`
963 
964  Returns
965  -------
966  exitStatus: `list` with `pipeBase.Struct`
967  exitStatus (0: success; 1: failure)
968  """
969 
970  task = self.TaskClass(config=self.config, log=self.log)
971 
972  exitStatus = 0
973  if self.doRaise:
974  task.runDataRef(butler)
975  else:
976  try:
977  task.runDataRef(butler)
978  except Exception as e:
979  exitStatus = 1
980  task.log.fatal("Failed: %s" % e)
981  if not isinstance(e, pipeBase.TaskError):
982  traceback.print_exc(file=sys.stderr)
983 
984  task.writeMetadata(butler)
985 
986  # The task does not return any results:
987  return [pipeBase.Struct(exitStatus=exitStatus)]
988 
989  def run(self, parsedCmd):
990  """
991  Run the task, with no multiprocessing
992 
993  Parameters
994  ----------
995  parsedCmd: ArgumentParser parsed command line
996  """
997 
998  resultList = []
999 
1000  if self.precall(parsedCmd):
1001  targetList = self.getTargetList(parsedCmd)
1002  # make sure that we only get 1
1003  resultList = self(targetList[0])
1004 
1005  return resultList
1006 
1007 
1008 class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
1009  """
1010  Run Single fit cycle for FGCM global calibration
1011  """
1012 
1013  ConfigClass = FgcmFitCycleConfig
1014  RunnerClass = FgcmFitCycleRunner
1015  _DefaultName = "fgcmFitCycle"
1016 
1017  def __init__(self, butler=None, initInputs=None, **kwargs):
1018  super().__init__(**kwargs)
1019 
1020  # no saving of metadata for now
1021  def _getMetadataName(self):
1022  return None
1023 
1024  def runQuantum(self, butlerQC, inputRefs, outputRefs):
1025  camera = butlerQC.get(inputRefs.camera)
1026 
1027  dataRefDict = {}
1028 
1029  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
1030  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
1031  dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
1032  dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
1033  dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
1034  if self.config.doReferenceCalibration:
1035  dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
1036  if self.config.cycleNumber > 0:
1037  dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
1038  dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
1039 
1040  fgcmDatasetDict = None
1041  if self.config.doMultipleCycles:
1042  # Run multiple cycles at once.
1043  config = copy.copy(self.config)
1044  config.update(cycleNumber=0)
1045  for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
1046  if cycle == self.config.multipleCyclesFinalCycleNumber:
1047  config.update(isFinalCycle=True)
1048 
1049  if cycle > 0:
1050  dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
1051  dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
1052 
1053  fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
1054  butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
1055  getattr(outputRefs, f'fgcmFitParameters{cycle}'))
1056  butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
1057  getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
1058  if self.outputZeropoints:
1059  butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
1060  getattr(outputRefs, f'fgcmZeropoints{cycle}'))
1061  butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
1062  getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
1063  if self.outputStandards:
1064  butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
1065  getattr(outputRefs, f'fgcmStandardStars{cycle}'))
1066  else:
1067  # Run a single cycle
1068  fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
1069 
1070  butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
1071  butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
1072  if self.outputZeropoints:
1073  butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
1074  butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
1075  if self.outputStandards:
1076  butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
1077 
1078  @pipeBase.timeMethod
1079  def runDataRef(self, butler):
1080  """
1081  Run a single fit cycle for FGCM
1082 
1083  Parameters
1084  ----------
1085  butler: `lsst.daf.persistence.Butler`
1086  """
1087  self._checkDatasetsExist(butler)
1088 
1089  dataRefDict = {}
1090  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
1091  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
1092  dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
1093  dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
1094  dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
1095  if self.config.doReferenceCalibration:
1096  dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
1097  if self.config.cycleNumber > 0:
1098  lastCycle = self.config.cycleNumber - 1
1099  dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
1100  fgcmcycle=lastCycle)
1101  dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
1102  fgcmcycle=lastCycle)
1103 
1104  camera = butler.get('camera')
1105  fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
1106 
1107  butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
1108  fgcmcycle=self.config.cycleNumber)
1109  butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
1110  fgcmcycle=self.config.cycleNumber)
1111  if self.outputZeropoints:
1112  butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
1113  fgcmcycle=self.config.cycleNumber)
1114  butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
1115  fgcmcycle=self.config.cycleNumber)
1116  if self.outputStandards:
1117  butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
1118  fgcmcycle=self.config.cycleNumber)
1119 
1120  def writeConfig(self, butler, clobber=False, doBackup=True):
1121  """Write the configuration used for processing the data, or check that an existing
1122  one is equal to the new one if present. This is an override of the regular
1123  version from pipe_base that knows about fgcmcycle.
1124 
1125  Parameters
1126  ----------
1127  butler : `lsst.daf.persistence.Butler`
1128  Data butler used to write the config. The config is written to dataset type
1129  `CmdLineTask._getConfigName`.
1130  clobber : `bool`, optional
1131  A boolean flag that controls what happens if a config already has been saved:
1132  - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1133  - `False`: raise `TaskError` if this config does not match the existing config.
1134  doBackup : `bool`, optional
1135  Set to `True` to backup the config files if clobbering.
1136  """
1137  configName = self._getConfigName()
1138  if configName is None:
1139  return
1140  if clobber:
1141  butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1142  elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1143  # this may be subject to a race condition; see #2789
1144  try:
1145  oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1146  except Exception as exc:
1147  raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1148  (configName, exc))
1149 
1150  def logConfigMismatch(msg):
1151  self.log.fatal("Comparing configuration: %s", msg)
1152 
1153  if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1154  raise pipeBase.TaskError(
1155  f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1156  " must be consistent within the same output repo (override with --clobber-config)")
1157  else:
1158  butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1159 
1160  def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1161  """
1162  Run the fit cycle
1163 
1164  Parameters
1165  ----------
1166  camera : `lsst.afw.cameraGeom.Camera`
1167  dataRefDict : `dict`
1168  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1169  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1170  dataRef dictionary with keys:
1171 
1172  ``"fgcmLookUpTable"``
1173  dataRef for the FGCM look-up table.
1174  ``"fgcmVisitCatalog"``
1175  dataRef for visit summary catalog.
1176  ``"fgcmStarObservations"``
1177  dataRef for star observation catalog.
1178  ``"fgcmStarIds"``
1179  dataRef for star id catalog.
1180  ``"fgcmStarIndices"``
1181  dataRef for star index catalog.
1182  ``"fgcmReferenceStars"``
1183  dataRef for matched reference star catalog.
1184  ``"fgcmFlaggedStars"``
1185  dataRef for flagged star catalog.
1186  ``"fgcmFitParameters"``
1187  dataRef for fit parameter catalog.
1188  config : `lsst.pex.config.Config`, optional
1189  Configuration to use to override self.config.
1190 
1191  Returns
1192  -------
1193  fgcmDatasetDict : `dict`
1194  Dictionary of datasets to persist.
1195  """
1196  if config is not None:
1197  _config = config
1198  else:
1199  _config = self.config
1200 
1201  # Set defaults on whether to output standards and zeropoints
1202  self.maxIter = _config.maxIterBeforeFinalCycle
1203  self.outputStandards = _config.outputStandardsBeforeFinalCycle
1204  self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1205  self.resetFitParameters = True
1206 
1207  if _config.isFinalCycle:
1208  # This is the final fit cycle, so we do not want to reset fit
1209  # parameters, we want to run a final "clean-up" with 0 fit iterations,
1210  # and we always want to output standards and zeropoints
1211  self.maxIter = 0
1212  self.outputStandards = True
1213  self.outputZeropoints = True
1214  self.resetFitParameters = False
1215 
1216  lutCat = dataRefDict['fgcmLookUpTable'].get()
1217  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1218  dict(_config.physicalFilterMap))
1219  del lutCat
1220 
1221  configDict = makeConfigDict(_config, self.log, camera,
1222  self.maxIter, self.resetFitParameters,
1223  self.outputZeropoints,
1224  lutIndexVals[0]['FILTERNAMES'])
1225 
1226  # next we need the exposure/visit information
1227  visitCat = dataRefDict['fgcmVisitCatalog'].get()
1228  fgcmExpInfo = translateVisitCatalog(visitCat)
1229  del visitCat
1230 
1231  # Use the first orientation.
1232  # TODO: DM-21215 will generalize to arbitrary camera orientations
1233  ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1234 
1235  noFitsDict = {'lutIndex': lutIndexVals,
1236  'lutStd': lutStd,
1237  'expInfo': fgcmExpInfo,
1238  'ccdOffsets': ccdOffsets}
1239 
1240  # set up the fitter object
1241  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1242  noFitsDict=noFitsDict, noOutput=True)
1243 
1244  # create the parameter object
1245  if (fgcmFitCycle.initialCycle):
1246  # cycle = 0, initial cycle
1247  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1248  fgcmLut,
1249  fgcmExpInfo)
1250  else:
1251  if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog):
1252  parCat = dataRefDict['fgcmFitParameters']
1253  else:
1254  parCat = dataRefDict['fgcmFitParameters'].get()
1255  inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1256  del parCat
1257  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1258  fgcmExpInfo,
1259  inParInfo,
1260  inParams,
1261  inSuperStar)
1262 
1263  # set up the stars...
1264  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1265 
1266  starObs = dataRefDict['fgcmStarObservations'].get()
1267  starIds = dataRefDict['fgcmStarIds'].get()
1268  starIndices = dataRefDict['fgcmStarIndices'].get()
1269 
1270  # grab the flagged stars if available
1271  if 'fgcmFlaggedStars' in dataRefDict:
1272  if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1273  flaggedStars = dataRefDict['fgcmFlaggedStars']
1274  else:
1275  flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1276  flagId = flaggedStars['objId'][:]
1277  flagFlag = flaggedStars['objFlag'][:]
1278  else:
1279  flaggedStars = None
1280  flagId = None
1281  flagFlag = None
1282 
1283  if _config.doReferenceCalibration:
1284  refStars = dataRefDict['fgcmReferenceStars'].get()
1285 
1286  refMag, refMagErr = extractReferenceMags(refStars,
1287  _config.bands,
1288  _config.physicalFilterMap)
1289  refId = refStars['fgcm_id'][:]
1290  else:
1291  refStars = None
1292  refId = None
1293  refMag = None
1294  refMagErr = None
1295 
1296  # match star observations to visits
1297  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1298  # actually be transferred into fgcm using the indexing below.
1299  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1300 
1301  # The fgcmStars.loadStars method will copy all the star information into
1302  # special shared memory objects that will not blow up the memory usage when
1303  # used with python multiprocessing. Once all the numbers are copied,
1304  # it is necessary to release all references to the objects that previously
1305  # stored the data to ensure that the garbage collector can clear the memory,
1306  # and ensure that this memory is not copied when multiprocessing kicks in.
1307 
1308  # We determine the conversion from the native units (typically radians) to
1309  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1310  # numpy arrays rather than Angles, which would we approximately 600x slower.
1311  conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1312 
1313  fgcmStars.loadStars(fgcmPars,
1314  starObs['visit'][starIndices['obsIndex']],
1315  starObs['ccd'][starIndices['obsIndex']],
1316  starObs['ra'][starIndices['obsIndex']] * conv,
1317  starObs['dec'][starIndices['obsIndex']] * conv,
1318  starObs['instMag'][starIndices['obsIndex']],
1319  starObs['instMagErr'][starIndices['obsIndex']],
1320  fgcmExpInfo['FILTERNAME'][visitIndex],
1321  starIds['fgcm_id'][:],
1322  starIds['ra'][:],
1323  starIds['dec'][:],
1324  starIds['obsArrIndex'][:],
1325  starIds['nObs'][:],
1326  obsX=starObs['x'][starIndices['obsIndex']],
1327  obsY=starObs['y'][starIndices['obsIndex']],
1328  obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1329  psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1330  refID=refId,
1331  refMag=refMag,
1332  refMagErr=refMagErr,
1333  flagID=flagId,
1334  flagFlag=flagFlag,
1335  computeNobs=True)
1336 
1337  # Release all references to temporary objects holding star data (see above)
1338  del starObs
1339  del starIds
1340  del starIndices
1341  del flagId
1342  del flagFlag
1343  del flaggedStars
1344  del refStars
1345  del refId
1346  del refMag
1347  del refMagErr
1348 
1349  # and set the bits in the cycle object
1350  fgcmFitCycle.setLUT(fgcmLut)
1351  fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1352  fgcmFitCycle.setPars(fgcmPars)
1353 
1354  # finish the setup
1355  fgcmFitCycle.finishSetup()
1356 
1357  # and run
1358  fgcmFitCycle.run()
1359 
1360 
1363 
1364  fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1365 
1366  # Output the config for the next cycle
1367  # We need to make a copy since the input one has been frozen
1368 
1369  updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1370  i, b in enumerate(_config.bands)}
1371  updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1372  i, band in enumerate(_config.bands)}
1373 
1374  outConfig = copy.copy(_config)
1375  outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1376  precomputeSuperStarInitialCycle=False,
1377  freezeStdAtmosphere=False,
1378  expGrayPhotometricCutDict=updatedPhotometricCutDict,
1379  expGrayHighCutDict=updatedHighCutDict)
1380 
1381  outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1382  cycleNumber=str(_config.cycleNumber + 1))
1383 
1384  configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1385  outConfig.cycleNumber)
1386  outConfig.save(configFileName)
1387 
1388  if _config.isFinalCycle == 1:
1389  # We are done, ready to output products
1390  self.log.info("Everything is in place to run fgcmOutputProducts.py")
1391  else:
1392  self.log.info("Saved config for next cycle to %s" % (configFileName))
1393  self.log.info("Be sure to look at:")
1394  self.log.info(" config.expGrayPhotometricCut")
1395  self.log.info(" config.expGrayHighCut")
1396  self.log.info("If you are satisfied with the fit, please set:")
1397  self.log.info(" config.isFinalCycle = True")
1398 
1399  return fgcmDatasetDict, outConfig
1400 
1401  def _checkDatasetsExist(self, butler):
1402  """
1403  Check if necessary datasets exist to run fgcmFitCycle
1404 
1405  Parameters
1406  ----------
1407  butler: `lsst.daf.persistence.Butler`
1408 
1409  Raises
1410  ------
1411  RuntimeError
1412  If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1413  fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1414  If cycleNumber > 0, then also checks for fgcmFitParameters,
1415  fgcmFlaggedStars.
1416  """
1417 
1418  if not butler.datasetExists('fgcmVisitCatalog'):
1419  raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1420  if not butler.datasetExists('fgcmStarObservations'):
1421  raise RuntimeError("Could not find fgcmStarObservations in repo!")
1422  if not butler.datasetExists('fgcmStarIds'):
1423  raise RuntimeError("Could not find fgcmStarIds in repo!")
1424  if not butler.datasetExists('fgcmStarIndices'):
1425  raise RuntimeError("Could not find fgcmStarIndices in repo!")
1426  if not butler.datasetExists('fgcmLookUpTable'):
1427  raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1428 
1429  # Need additional datasets if we are not the initial cycle
1430  if (self.config.cycleNumber > 0):
1431  if not butler.datasetExists('fgcmFitParameters',
1432  fgcmcycle=self.config.cycleNumber-1):
1433  raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1434  (self.config.cycleNumber-1))
1435  if not butler.datasetExists('fgcmFlaggedStars',
1436  fgcmcycle=self.config.cycleNumber-1):
1437  raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1438  (self.config.cycleNumber-1))
1439 
1440  # And additional dataset if we want reference calibration
1441  if self.config.doReferenceCalibration:
1442  if not butler.datasetExists('fgcmReferenceStars'):
1443  raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1444  "doReferenceCalibration is True.")
1445 
1446  def _loadParameters(self, parCat):
1447  """
1448  Load FGCM parameters from a previous fit cycle
1449 
1450  Parameters
1451  ----------
1452  parCat : `lsst.afw.table.BaseCatalog`
1453  Parameter catalog in afw table form.
1454 
1455  Returns
1456  -------
1457  inParInfo: `numpy.ndarray`
1458  Numpy array parameter information formatted for input to fgcm
1459  inParameters: `numpy.ndarray`
1460  Numpy array parameter values formatted for input to fgcm
1461  inSuperStar: `numpy.array`
1462  Superstar flat formatted for input to fgcm
1463  """
1464  parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1465  parFitBands = np.array(parCat[0]['fitBands'].split(','))
1466 
1467  inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1468  ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1469  (parLutFilterNames.size, )),
1470  ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1471  ('LNTAUUNIT', 'f8'),
1472  ('LNTAUSLOPEUNIT', 'f8'),
1473  ('ALPHAUNIT', 'f8'),
1474  ('LNPWVUNIT', 'f8'),
1475  ('LNPWVSLOPEUNIT', 'f8'),
1476  ('LNPWVQUADRATICUNIT', 'f8'),
1477  ('LNPWVGLOBALUNIT', 'f8'),
1478  ('O3UNIT', 'f8'),
1479  ('QESYSUNIT', 'f8'),
1480  ('FILTEROFFSETUNIT', 'f8'),
1481  ('HASEXTERNALPWV', 'i2'),
1482  ('HASEXTERNALTAU', 'i2')])
1483  inParInfo['NCCD'] = parCat['nCcd']
1484  inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1485  inParInfo['FITBANDS'][:] = parFitBands
1486  inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1487  inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1488 
1489  inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1490  ('PARO3', 'f8', (parCat['parO3'].size, )),
1491  ('PARLNTAUINTERCEPT', 'f8',
1492  (parCat['parLnTauIntercept'].size, )),
1493  ('PARLNTAUSLOPE', 'f8',
1494  (parCat['parLnTauSlope'].size, )),
1495  ('PARLNPWVINTERCEPT', 'f8',
1496  (parCat['parLnPwvIntercept'].size, )),
1497  ('PARLNPWVSLOPE', 'f8',
1498  (parCat['parLnPwvSlope'].size, )),
1499  ('PARLNPWVQUADRATIC', 'f8',
1500  (parCat['parLnPwvQuadratic'].size, )),
1501  ('PARQESYSINTERCEPT', 'f8',
1502  (parCat['parQeSysIntercept'].size, )),
1503  ('COMPQESYSSLOPE', 'f8',
1504  (parCat['compQeSysSlope'].size, )),
1505  ('PARFILTEROFFSET', 'f8',
1506  (parCat['parFilterOffset'].size, )),
1507  ('PARFILTEROFFSETFITFLAG', 'i2',
1508  (parCat['parFilterOffsetFitFlag'].size, )),
1509  ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1510  ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1511  ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1512  (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1513  ('COMPABSTHROUGHPUT', 'f8',
1514  (parCat['compAbsThroughput'].size, )),
1515  ('COMPREFOFFSET', 'f8',
1516  (parCat['compRefOffset'].size, )),
1517  ('COMPREFSIGMA', 'f8',
1518  (parCat['compRefSigma'].size, )),
1519  ('COMPMIRRORCHROMATICITY', 'f8',
1520  (parCat['compMirrorChromaticity'].size, )),
1521  ('MIRRORCHROMATICITYPIVOT', 'f8',
1522  (parCat['mirrorChromaticityPivot'].size, )),
1523  ('COMPMEDIANSEDSLOPE', 'f8',
1524  (parCat['compMedianSedSlope'].size, )),
1525  ('COMPAPERCORRPIVOT', 'f8',
1526  (parCat['compAperCorrPivot'].size, )),
1527  ('COMPAPERCORRSLOPE', 'f8',
1528  (parCat['compAperCorrSlope'].size, )),
1529  ('COMPAPERCORRSLOPEERR', 'f8',
1530  (parCat['compAperCorrSlopeErr'].size, )),
1531  ('COMPAPERCORRRANGE', 'f8',
1532  (parCat['compAperCorrRange'].size, )),
1533  ('COMPMODELERREXPTIMEPIVOT', 'f8',
1534  (parCat['compModelErrExptimePivot'].size, )),
1535  ('COMPMODELERRFWHMPIVOT', 'f8',
1536  (parCat['compModelErrFwhmPivot'].size, )),
1537  ('COMPMODELERRSKYPIVOT', 'f8',
1538  (parCat['compModelErrSkyPivot'].size, )),
1539  ('COMPMODELERRPARS', 'f8',
1540  (parCat['compModelErrPars'].size, )),
1541  ('COMPEXPGRAY', 'f8',
1542  (parCat['compExpGray'].size, )),
1543  ('COMPVARGRAY', 'f8',
1544  (parCat['compVarGray'].size, )),
1545  ('COMPEXPDELTAMAGBKG', 'f8',
1546  (parCat['compExpDeltaMagBkg'].size, )),
1547  ('COMPNGOODSTARPEREXP', 'i4',
1548  (parCat['compNGoodStarPerExp'].size, )),
1549  ('COMPSIGFGCM', 'f8',
1550  (parCat['compSigFgcm'].size, )),
1551  ('COMPSIGMACAL', 'f8',
1552  (parCat['compSigmaCal'].size, )),
1553  ('COMPRETRIEVEDLNPWV', 'f8',
1554  (parCat['compRetrievedLnPwv'].size, )),
1555  ('COMPRETRIEVEDLNPWVRAW', 'f8',
1556  (parCat['compRetrievedLnPwvRaw'].size, )),
1557  ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1558  (parCat['compRetrievedLnPwvFlag'].size, )),
1559  ('COMPRETRIEVEDTAUNIGHT', 'f8',
1560  (parCat['compRetrievedTauNight'].size, ))])
1561 
1562  inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1563  inParams['PARO3'][:] = parCat['parO3'][0, :]
1564  inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1565  inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1566  inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1567  inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1568  inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1569  inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1570  inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1571  inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1572  inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1573  inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1574  inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1575  inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1576  inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1577  inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1578  inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1579  inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1580  inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1581  inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1582  inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1583  inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1584  inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1585  inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1586  inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1587  inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1588  inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1589  inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1590  inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1591  inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1592  inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1593  inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1594  inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1595  inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1596  inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1597  inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1598  inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1599  inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1600 
1601  inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1602  inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1603 
1604  return (inParInfo, inParams, inSuperStar)
1605 
1606  def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1607  """
1608  Persist FGCM datasets through the butler.
1609 
1610  Parameters
1611  ----------
1612  fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1613  Fgcm Fit cycle object
1614  """
1615  fgcmDatasetDict = {}
1616 
1617  # Save the parameters
1618  parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1619 
1620  parSchema = afwTable.Schema()
1621 
1622  comma = ','
1623  lutFilterNameString = comma.join([n.decode('utf-8')
1624  for n in parInfo['LUTFILTERNAMES'][0]])
1625  fitBandString = comma.join([n.decode('utf-8')
1626  for n in parInfo['FITBANDS'][0]])
1627 
1628  parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1629  lutFilterNameString, fitBandString)
1630  parCat = self._makeParCatalog(parSchema, parInfo, pars,
1631  fgcmFitCycle.fgcmPars.parSuperStarFlat,
1632  lutFilterNameString, fitBandString)
1633 
1634  fgcmDatasetDict['fgcmFitParameters'] = parCat
1635 
1636  # Save the indices of the flagged stars
1637  # (stars that have been (a) reserved from the fit for testing and
1638  # (b) bad stars that have failed quality checks.)
1639  flagStarSchema = self._makeFlagStarSchema()
1640  flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1641  flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1642 
1643  fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1644 
1645  # Save the zeropoint information and atmospheres only if desired
1646  if self.outputZeropoints:
1647  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1648  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1649 
1650  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1651  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1652 
1653  fgcmDatasetDict['fgcmZeropoints'] = zptCat
1654 
1655  # Save atmosphere values
1656  # These are generated by the same code that generates zeropoints
1657  atmSchema = makeAtmSchema()
1658  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1659 
1660  fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1661 
1662  # Save the standard stars (if configured)
1663  if self.outputStandards:
1664  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1665  stdSchema = makeStdSchema(len(goodBands))
1666  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1667 
1668  fgcmDatasetDict['fgcmStandardStars'] = stdCat
1669 
1670  return fgcmDatasetDict
1671 
1672  def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1673  lutFilterNameString, fitBandString):
1674  """
1675  Make the parameter persistence schema
1676 
1677  Parameters
1678  ----------
1679  parInfo: `numpy.ndarray`
1680  Parameter information returned by fgcm
1681  pars: `numpy.ndarray`
1682  Parameter values returned by fgcm
1683  parSuperStarFlat: `numpy.array`
1684  Superstar flat values returned by fgcm
1685  lutFilterNameString: `str`
1686  Combined string of all the lutFilterNames
1687  fitBandString: `str`
1688  Combined string of all the fitBands
1689 
1690  Returns
1691  -------
1692  parSchema: `afwTable.schema`
1693  """
1694 
1695  parSchema = afwTable.Schema()
1696 
1697  # parameter info section
1698  parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1699  parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1700  size=len(lutFilterNameString))
1701  parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1702  size=len(fitBandString))
1703  parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1704  parSchema.addField('lnTauSlopeUnit', type=np.float64,
1705  doc='Step units for ln(AOD) slope')
1706  parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1707  parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1708  parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1709  doc='Step units for ln(pwv) slope')
1710  parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1711  doc='Step units for ln(pwv) quadratic term')
1712  parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1713  doc='Step units for global ln(pwv) parameters')
1714  parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1715  parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1716  parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1717  parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1718  parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1719 
1720  # parameter section
1721  parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1722  size=pars['PARALPHA'].size)
1723  parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1724  size=pars['PARO3'].size)
1725  parSchema.addField('parLnTauIntercept', type='ArrayD',
1726  doc='ln(Tau) intercept parameter vector',
1727  size=pars['PARLNTAUINTERCEPT'].size)
1728  parSchema.addField('parLnTauSlope', type='ArrayD',
1729  doc='ln(Tau) slope parameter vector',
1730  size=pars['PARLNTAUSLOPE'].size)
1731  parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1732  size=pars['PARLNPWVINTERCEPT'].size)
1733  parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1734  size=pars['PARLNPWVSLOPE'].size)
1735  parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1736  size=pars['PARLNPWVQUADRATIC'].size)
1737  parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1738  size=pars['PARQESYSINTERCEPT'].size)
1739  parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1740  size=pars[0]['COMPQESYSSLOPE'].size)
1741  parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1742  size=pars['PARFILTEROFFSET'].size)
1743  parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1744  size=pars['PARFILTEROFFSETFITFLAG'].size)
1745  parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1746  doc='Global scale for retrieved ln(pwv)')
1747  parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1748  doc='Global offset for retrieved ln(pwv)')
1749  parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1750  doc='Nightly offset for retrieved ln(pwv)',
1751  size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1752  parSchema.addField('compAbsThroughput', type='ArrayD',
1753  doc='Absolute throughput (relative to transmission curves)',
1754  size=pars['COMPABSTHROUGHPUT'].size)
1755  parSchema.addField('compRefOffset', type='ArrayD',
1756  doc='Offset between reference stars and calibrated stars',
1757  size=pars['COMPREFOFFSET'].size)
1758  parSchema.addField('compRefSigma', type='ArrayD',
1759  doc='Width of reference star/calibrated star distribution',
1760  size=pars['COMPREFSIGMA'].size)
1761  parSchema.addField('compMirrorChromaticity', type='ArrayD',
1762  doc='Computed mirror chromaticity terms',
1763  size=pars['COMPMIRRORCHROMATICITY'].size)
1764  parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1765  doc='Mirror chromaticity pivot mjd',
1766  size=pars['MIRRORCHROMATICITYPIVOT'].size)
1767  parSchema.addField('compMedianSedSlope', type='ArrayD',
1768  doc='Computed median SED slope (per band)',
1769  size=pars['COMPMEDIANSEDSLOPE'].size)
1770  parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1771  size=pars['COMPAPERCORRPIVOT'].size)
1772  parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1773  size=pars['COMPAPERCORRSLOPE'].size)
1774  parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1775  size=pars['COMPAPERCORRSLOPEERR'].size)
1776  parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1777  size=pars['COMPAPERCORRRANGE'].size)
1778  parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1779  size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1780  parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1781  size=pars['COMPMODELERRFWHMPIVOT'].size)
1782  parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1783  size=pars['COMPMODELERRSKYPIVOT'].size)
1784  parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1785  size=pars['COMPMODELERRPARS'].size)
1786  parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1787  size=pars['COMPEXPGRAY'].size)
1788  parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1789  size=pars['COMPVARGRAY'].size)
1790  parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1791  doc='Computed exposure offset due to background',
1792  size=pars['COMPEXPDELTAMAGBKG'].size)
1793  parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1794  doc='Computed number of good stars per exposure',
1795  size=pars['COMPNGOODSTARPEREXP'].size)
1796  parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1797  size=pars['COMPSIGFGCM'].size)
1798  parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1799  size=pars['COMPSIGMACAL'].size)
1800  parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1801  size=pars['COMPRETRIEVEDLNPWV'].size)
1802  parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1803  size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1804  parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1805  size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1806  parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1807  size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1808  # superstarflat section
1809  parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1810  size=4)
1811  parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1812  size=parSuperStarFlat.size)
1813 
1814  return parSchema
1815 
1816  def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1817  lutFilterNameString, fitBandString):
1818  """
1819  Make the FGCM parameter catalog for persistence
1820 
1821  Parameters
1822  ----------
1823  parSchema: `lsst.afw.table.Schema`
1824  Parameter catalog schema
1825  pars: `numpy.ndarray`
1826  FGCM parameters to put into parCat
1827  parSuperStarFlat: `numpy.array`
1828  FGCM superstar flat array to put into parCat
1829  lutFilterNameString: `str`
1830  Combined string of all the lutFilterNames
1831  fitBandString: `str`
1832  Combined string of all the fitBands
1833 
1834  Returns
1835  -------
1836  parCat: `afwTable.BasicCatalog`
1837  Atmosphere and instrumental model parameter catalog for persistence
1838  """
1839 
1840  parCat = afwTable.BaseCatalog(parSchema)
1841  parCat.reserve(1)
1842 
1843  # The parameter catalog just has one row, with many columns for all the
1844  # atmosphere and instrument fit parameters
1845  rec = parCat.addNew()
1846 
1847  # info section
1848  rec['nCcd'] = parInfo['NCCD']
1849  rec['lutFilterNames'] = lutFilterNameString
1850  rec['fitBands'] = fitBandString
1851  # note these are not currently supported here.
1852  rec['hasExternalPwv'] = 0
1853  rec['hasExternalTau'] = 0
1854 
1855  # parameter section
1856 
1857  scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1858 
1859  arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1860  'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1861  'parQeSysIntercept', 'compQeSysSlope',
1862  'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1863  'parFilterOffset', 'parFilterOffsetFitFlag',
1864  'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1865  'compMirrorChromaticity', 'mirrorChromaticityPivot',
1866  'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1867  'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1868  'compModelErrSkyPivot', 'compModelErrPars',
1869  'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1870  'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1871  'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1872  'compRetrievedTauNight']
1873 
1874  for scalarName in scalarNames:
1875  rec[scalarName] = pars[scalarName.upper()]
1876 
1877  for arrName in arrNames:
1878  rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1879 
1880  # superstar section
1881  rec['superstarSize'][:] = parSuperStarFlat.shape
1882  rec['superstar'][:] = parSuperStarFlat.flatten()
1883 
1884  return parCat
1885 
1886  def _makeFlagStarSchema(self):
1887  """
1888  Make the flagged-stars schema
1889 
1890  Returns
1891  -------
1892  flagStarSchema: `lsst.afw.table.Schema`
1893  """
1894 
1895  flagStarSchema = afwTable.Schema()
1896 
1897  flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1898  flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1899 
1900  return flagStarSchema
1901 
1902  def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1903  """
1904  Make the flagged star catalog for persistence
1905 
1906  Parameters
1907  ----------
1908  flagStarSchema: `lsst.afw.table.Schema`
1909  Flagged star schema
1910  flagStarStruct: `numpy.ndarray`
1911  Flagged star structure from fgcm
1912 
1913  Returns
1914  -------
1915  flagStarCat: `lsst.afw.table.BaseCatalog`
1916  Flagged star catalog for persistence
1917  """
1918 
1919  flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1920  flagStarCat.resize(flagStarStruct.size)
1921 
1922  flagStarCat['objId'][:] = flagStarStruct['OBJID']
1923  flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1924 
1925  return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:882
def makeStdSchema(nBands)
Definition: utilities.py:737
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:704
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:50
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:217
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:625
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:769
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:539
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:398
def translateVisitCatalog(visitCat)
Definition: utilities.py:347