lsst.fgcmcal  22.0.1-14-g6272d78+ec7fbbaa4d
fgcmFitCycle.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Perform a single fit cycle of FGCM.
24 
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
28 for the global fit).
29 
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
34 """
35 
36 import sys
37 import traceback
38 import copy
39 
40 import numpy as np
41 
42 import lsst.pex.config as pexConfig
43 import lsst.pipe.base as pipeBase
44 from lsst.pipe.base import connectionTypes
45 import lsst.afw.table as afwTable
46 from lsst.utils.timer import timeMethod
47 
48 from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
49 from .utilities import extractReferenceMags
50 from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
51 from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
52 from .sedterms import SedboundarytermDict, SedtermDict
53 from .utilities import lookupStaticCalibrations
54 
55 import fgcm
56 
57 __all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
58 
59 MULTIPLE_CYCLES_MAX = 10
60 
61 
62 class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
63  dimensions=("instrument",),
64  defaultTemplates={"previousCycleNumber": "-1",
65  "cycleNumber": "0"}):
66  camera = connectionTypes.PrerequisiteInput(
67  doc="Camera instrument",
68  name="camera",
69  storageClass="Camera",
70  dimensions=("instrument",),
71  lookupFunction=lookupStaticCalibrations,
72  isCalibration=True,
73  )
74 
75  fgcmLookUpTable = connectionTypes.PrerequisiteInput(
76  doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
77  "chromatic corrections."),
78  name="fgcmLookUpTable",
79  storageClass="Catalog",
80  dimensions=("instrument",),
81  deferLoad=True,
82  )
83 
84  fgcmVisitCatalog = connectionTypes.Input(
85  doc="Catalog of visit information for fgcm",
86  name="fgcmVisitCatalog",
87  storageClass="Catalog",
88  dimensions=("instrument",),
89  deferLoad=True,
90  )
91 
92  fgcmStarObservations = connectionTypes.Input(
93  doc="Catalog of star observations for fgcm",
94  name="fgcmStarObservations",
95  storageClass="Catalog",
96  dimensions=("instrument",),
97  deferLoad=True,
98  )
99 
100  fgcmStarIds = connectionTypes.Input(
101  doc="Catalog of fgcm calibration star IDs",
102  name="fgcmStarIds",
103  storageClass="Catalog",
104  dimensions=("instrument",),
105  deferLoad=True,
106  )
107 
108  fgcmStarIndices = connectionTypes.Input(
109  doc="Catalog of fgcm calibration star indices",
110  name="fgcmStarIndices",
111  storageClass="Catalog",
112  dimensions=("instrument",),
113  deferLoad=True,
114  )
115 
116  fgcmReferenceStars = connectionTypes.Input(
117  doc="Catalog of fgcm-matched reference stars",
118  name="fgcmReferenceStars",
119  storageClass="Catalog",
120  dimensions=("instrument",),
121  deferLoad=True,
122  )
123 
124  fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
125  doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
126  name="fgcmFlaggedStars{previousCycleNumber}",
127  storageClass="Catalog",
128  dimensions=("instrument",),
129  deferLoad=True,
130  )
131 
132  fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
133  doc="Catalog of fgcm fit parameters from previous fit cycle",
134  name="fgcmFitParameters{previousCycleNumber}",
135  storageClass="Catalog",
136  dimensions=("instrument",),
137  deferLoad=True,
138  )
139 
140  fgcmFitParameters = connectionTypes.Output(
141  doc="Catalog of fgcm fit parameters from current fit cycle",
142  name="fgcmFitParameters{cycleNumber}",
143  storageClass="Catalog",
144  dimensions=("instrument",),
145  )
146 
147  fgcmFlaggedStars = connectionTypes.Output(
148  doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
149  name="fgcmFlaggedStars{cycleNumber}",
150  storageClass="Catalog",
151  dimensions=("instrument",),
152  )
153 
154  fgcmZeropoints = connectionTypes.Output(
155  doc="Catalog of fgcm zeropoint data from current fit cycle",
156  name="fgcmZeropoints{cycleNumber}",
157  storageClass="Catalog",
158  dimensions=("instrument",),
159  )
160 
161  fgcmAtmosphereParameters = connectionTypes.Output(
162  doc="Catalog of atmospheric fit parameters from current fit cycle",
163  name="fgcmAtmosphereParameters{cycleNumber}",
164  storageClass="Catalog",
165  dimensions=("instrument",),
166  )
167 
168  fgcmStandardStars = connectionTypes.Output(
169  doc="Catalog of standard star magnitudes from current fit cycle",
170  name="fgcmStandardStars{cycleNumber}",
171  storageClass="SimpleCatalog",
172  dimensions=("instrument",),
173  )
174 
175  # Add connections for running multiple cycles
176  # This uses vars() to alter the class __dict__ to programmatically
177  # write many similar outputs.
178  for cycle in range(MULTIPLE_CYCLES_MAX):
179  vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
180  doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
181  name=f"fgcmFitParameters{cycle}",
182  storageClass="Catalog",
183  dimensions=("instrument",),
184  )
185  vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
186  doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
187  name=f"fgcmFlaggedStars{cycle}",
188  storageClass="Catalog",
189  dimensions=("instrument",),
190  )
191  vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
192  doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
193  name=f"fgcmZeropoints{cycle}",
194  storageClass="Catalog",
195  dimensions=("instrument",),
196  )
197  vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
198  doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
199  name=f"fgcmAtmosphereParameters{cycle}",
200  storageClass="Catalog",
201  dimensions=("instrument",),
202  )
203  vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
204  doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
205  name=f"fgcmStandardStars{cycle}",
206  storageClass="SimpleCatalog",
207  dimensions=("instrument",),
208  )
209 
210  def __init__(self, *, config=None):
211  super().__init__(config=config)
212 
213  if not config.doReferenceCalibration:
214  self.inputs.remove("fgcmReferenceStars")
215 
216  if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
217  raise ValueError("cycleNumber must be of integer format")
218  if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
219  raise ValueError("previousCycleNumber must be of integer format")
220  if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
221  raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
222 
223  if int(config.connections.cycleNumber) == 0:
224  self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
225  self.prerequisiteInputs.remove("fgcmFitParametersInput")
226 
227  if not self.config.doMultipleCycles:
228  # Single-cycle run
229  if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
230  self.outputs.remove("fgcmStandardStars")
231 
232  if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
233  self.outputs.remove("fgcmZeropoints")
234  self.outputs.remove("fgcmAtmosphereParameters")
235 
236  # Remove all multiple cycle outputs
237  for cycle in range(0, MULTIPLE_CYCLES_MAX):
238  self.outputs.remove(f"fgcmFitParameters{cycle}")
239  self.outputs.remove(f"fgcmFlaggedStars{cycle}")
240  self.outputs.remove(f"fgcmZeropoints{cycle}")
241  self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
242  self.outputs.remove(f"fgcmStandardStars{cycle}")
243 
244  else:
245  # Multiple-cycle run
246  # Remove single-cycle outputs
247  self.outputs.remove("fgcmFitParameters")
248  self.outputs.remove("fgcmFlaggedStars")
249  self.outputs.remove("fgcmZeropoints")
250  self.outputs.remove("fgcmAtmosphereParameters")
251  self.outputs.remove("fgcmStandardStars")
252 
253  # Remove outputs from cycles that are not used
254  for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
255  MULTIPLE_CYCLES_MAX):
256  self.outputs.remove(f"fgcmFitParameters{cycle}")
257  self.outputs.remove(f"fgcmFlaggedStars{cycle}")
258  self.outputs.remove(f"fgcmZeropoints{cycle}")
259  self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
260  self.outputs.remove(f"fgcmStandardStars{cycle}")
261 
262  # Remove non-final-cycle outputs if necessary
263  for cycle in range(self.config.multipleCyclesFinalCycleNumber):
264  if not self.config.outputZeropointsBeforeFinalCycle:
265  self.outputs.remove(f"fgcmZeropoints{cycle}")
266  self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
267  if not self.config.outputStandardsBeforeFinalCycle:
268  self.outputs.remove(f"fgcmStandardStars{cycle}")
269 
270 
271 class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
272  pipelineConnections=FgcmFitCycleConnections):
273  """Config for FgcmFitCycle"""
274 
275  doMultipleCycles = pexConfig.Field(
276  doc="Run multiple fit cycles in one task",
277  dtype=bool,
278  default=False,
279  )
280  multipleCyclesFinalCycleNumber = pexConfig.RangeField(
281  doc=("Final cycle number in multiple cycle mode. The initial cycle "
282  "is 0, with limited parameters fit. The next cycle is 1 with "
283  "full parameter fit. The final cycle is a clean-up with no "
284  "parameters fit. There will be a total of "
285  "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
286  "cycle number cannot be less than 2."),
287  dtype=int,
288  default=5,
289  min=2,
290  max=MULTIPLE_CYCLES_MAX,
291  inclusiveMax=True,
292  )
293  bands = pexConfig.ListField(
294  doc="Bands to run calibration",
295  dtype=str,
296  default=[],
297  )
298  fitBands = pexConfig.ListField(
299  doc=("Bands to use in atmospheric fit. The bands not listed here will have "
300  "the atmosphere constrained from the 'fitBands' on the same night. "
301  "Must be a subset of `config.bands`"),
302  dtype=str,
303  default=[],
304  )
305  requiredBands = pexConfig.ListField(
306  doc=("Bands that are required for a star to be considered a calibration star. "
307  "Must be a subset of `config.bands`"),
308  dtype=str,
309  default=[],
310  )
311  # The following config will not be necessary after Gen2 retirement.
312  # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
313  # is easiest to access in the config file.
314  physicalFilterMap = pexConfig.DictField(
315  doc="Mapping from 'physicalFilter' to band.",
316  keytype=str,
317  itemtype=str,
318  default={},
319  )
320  doReferenceCalibration = pexConfig.Field(
321  doc="Use reference catalog as additional constraint on calibration",
322  dtype=bool,
323  default=True,
324  )
325  refStarSnMin = pexConfig.Field(
326  doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
327  dtype=float,
328  default=50.0,
329  )
330  refStarOutlierNSig = pexConfig.Field(
331  doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
332  "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
333  dtype=float,
334  default=4.0,
335  )
336  applyRefStarColorCuts = pexConfig.Field(
337  doc="Apply color cuts to reference stars?",
338  dtype=bool,
339  default=True,
340  )
341  nCore = pexConfig.Field(
342  doc="Number of cores to use",
343  dtype=int,
344  default=4,
345  )
346  nStarPerRun = pexConfig.Field(
347  doc="Number of stars to run in each chunk",
348  dtype=int,
349  default=200000,
350  )
351  nExpPerRun = pexConfig.Field(
352  doc="Number of exposures to run in each chunk",
353  dtype=int,
354  default=1000,
355  )
356  reserveFraction = pexConfig.Field(
357  doc="Fraction of stars to reserve for testing",
358  dtype=float,
359  default=0.1,
360  )
361  freezeStdAtmosphere = pexConfig.Field(
362  doc="Freeze atmosphere parameters to standard (for testing)",
363  dtype=bool,
364  default=False,
365  )
366  precomputeSuperStarInitialCycle = pexConfig.Field(
367  doc="Precompute superstar flat for initial cycle",
368  dtype=bool,
369  default=False,
370  )
371  superStarSubCcdDict = pexConfig.DictField(
372  doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
373  "Must have one entry per band."),
374  keytype=str,
375  itemtype=bool,
376  default={},
377  )
378  superStarSubCcdChebyshevOrder = pexConfig.Field(
379  doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
380  "Global default is first-order polynomials, and should be overridden "
381  "on a camera-by-camera basis depending on the ISR."),
382  dtype=int,
383  default=1,
384  )
385  superStarSubCcdTriangular = pexConfig.Field(
386  doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
387  "suppress high-order cross terms?"),
388  dtype=bool,
389  default=False,
390  )
391  superStarSigmaClip = pexConfig.Field(
392  doc="Number of sigma to clip outliers when selecting for superstar flats",
393  dtype=float,
394  default=5.0,
395  )
396  focalPlaneSigmaClip = pexConfig.Field(
397  doc="Number of sigma to clip outliers per focal-plane.",
398  dtype=float,
399  default=4.0,
400  )
401  ccdGraySubCcdDict = pexConfig.DictField(
402  doc=("Per-band specification on whether to compute achromatic per-ccd residual "
403  "('ccd gray') on a sub-ccd scale."),
404  keytype=str,
405  itemtype=bool,
406  default={},
407  )
408  ccdGraySubCcdChebyshevOrder = pexConfig.Field(
409  doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
410  dtype=int,
411  default=1,
412  )
413  ccdGraySubCcdTriangular = pexConfig.Field(
414  doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
415  "suppress high-order cross terms?"),
416  dtype=bool,
417  default=True,
418  )
419  ccdGrayFocalPlaneDict = pexConfig.DictField(
420  doc=("Per-band specification on whether to compute focal-plane residual "
421  "('ccd gray') corrections."),
422  keytype=str,
423  itemtype=bool,
424  default={},
425  )
426  ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
427  doc=("Minimum number of 'good' CCDs required to perform focal-plane "
428  "gray corrections. If there are fewer good CCDs then the gray "
429  "correction is computed per-ccd."),
430  dtype=int,
431  default=1,
432  )
433  ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
434  doc="Order of the 2D chebyshev polynomials for focal plane fit.",
435  dtype=int,
436  default=3,
437  )
438  cycleNumber = pexConfig.Field(
439  doc=("FGCM fit cycle number. This is automatically incremented after each run "
440  "and stage of outlier rejection. See cookbook for details."),
441  dtype=int,
442  default=None,
443  )
444  isFinalCycle = pexConfig.Field(
445  doc=("Is this the final cycle of the fitting? Will automatically compute final "
446  "selection of stars and photometric exposures, and will output zeropoints "
447  "and standard stars for use in fgcmOutputProducts"),
448  dtype=bool,
449  default=False,
450  )
451  maxIterBeforeFinalCycle = pexConfig.Field(
452  doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
453  "will always be 0 in the final cycle for cleanup and final selection."),
454  dtype=int,
455  default=50,
456  )
457  deltaMagBkgOffsetPercentile = pexConfig.Field(
458  doc=("Percentile brightest stars on a visit/ccd to use to compute net "
459  "offset from local background subtraction."),
460  dtype=float,
461  default=0.25,
462  )
463  deltaMagBkgPerCcd = pexConfig.Field(
464  doc=("Compute net offset from local background subtraction per-ccd? "
465  "Otherwise, use computation per visit."),
466  dtype=bool,
467  default=False,
468  )
469  utBoundary = pexConfig.Field(
470  doc="Boundary (in UTC) from day-to-day",
471  dtype=float,
472  default=None,
473  )
474  washMjds = pexConfig.ListField(
475  doc="Mirror wash MJDs",
476  dtype=float,
477  default=(0.0,),
478  )
479  epochMjds = pexConfig.ListField(
480  doc="Epoch boundaries in MJD",
481  dtype=float,
482  default=(0.0,),
483  )
484  minObsPerBand = pexConfig.Field(
485  doc="Minimum good observations per band",
486  dtype=int,
487  default=2,
488  )
489  # TODO: When DM-16511 is done, it will be possible to get the
490  # telescope latitude directly from the camera.
491  latitude = pexConfig.Field(
492  doc="Observatory latitude",
493  dtype=float,
494  default=None,
495  )
496  brightObsGrayMax = pexConfig.Field(
497  doc="Maximum gray extinction to be considered bright observation",
498  dtype=float,
499  default=0.15,
500  )
501  minStarPerCcd = pexConfig.Field(
502  doc=("Minimum number of good stars per CCD to be used in calibration fit. "
503  "CCDs with fewer stars will have their calibration estimated from other "
504  "CCDs in the same visit, with zeropoint error increased accordingly."),
505  dtype=int,
506  default=5,
507  )
508  minCcdPerExp = pexConfig.Field(
509  doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
510  "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
511  dtype=int,
512  default=5,
513  )
514  maxCcdGrayErr = pexConfig.Field(
515  doc="Maximum error on CCD gray offset to be considered photometric",
516  dtype=float,
517  default=0.05,
518  )
519  minStarPerExp = pexConfig.Field(
520  doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
521  "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
522  dtype=int,
523  default=600,
524  )
525  minExpPerNight = pexConfig.Field(
526  doc="Minimum number of good exposures/visits to consider a partly photometric night",
527  dtype=int,
528  default=10,
529  )
530  expGrayInitialCut = pexConfig.Field(
531  doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
532  "observations."),
533  dtype=float,
534  default=-0.25,
535  )
536  expGrayPhotometricCutDict = pexConfig.DictField(
537  doc=("Per-band specification on maximum (negative) achromatic exposure residual "
538  "('gray term') for a visit to be considered photometric. Must have one "
539  "entry per band. Broad-band filters should be -0.05."),
540  keytype=str,
541  itemtype=float,
542  default={},
543  )
544  expGrayHighCutDict = pexConfig.DictField(
545  doc=("Per-band specification on maximum (positive) achromatic exposure residual "
546  "('gray term') for a visit to be considered photometric. Must have one "
547  "entry per band. Broad-band filters should be 0.2."),
548  keytype=str,
549  itemtype=float,
550  default={},
551  )
552  expGrayRecoverCut = pexConfig.Field(
553  doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
554  "Visits with more gray extinction will only get CCD zeropoints if there are "
555  "sufficient star observations (minStarPerCcd) on that CCD."),
556  dtype=float,
557  default=-1.0,
558  )
559  expVarGrayPhotometricCutDict = pexConfig.DictField(
560  doc=("Per-band specification on maximum exposure variance to be considered possibly "
561  "photometric. Must have one entry per band. Broad-band filters should be "
562  "0.0005."),
563  keytype=str,
564  itemtype=float,
565  default={},
566  )
567  expGrayErrRecoverCut = pexConfig.Field(
568  doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
569  "Visits with more gray variance will only get CCD zeropoints if there are "
570  "sufficient star observations (minStarPerCcd) on that CCD."),
571  dtype=float,
572  default=0.05,
573  )
574  aperCorrFitNBins = pexConfig.Field(
575  doc=("Number of aperture bins used in aperture correction fit. When set to 0"
576  "no fit will be performed, and the config.aperCorrInputSlopes will be "
577  "used if available."),
578  dtype=int,
579  default=10,
580  )
581  aperCorrInputSlopeDict = pexConfig.DictField(
582  doc=("Per-band specification of aperture correction input slope parameters. These "
583  "are used on the first fit iteration, and aperture correction parameters will "
584  "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
585  "to set this when there is insufficient data to fit the parameters (e.g. "
586  "tract mode)."),
587  keytype=str,
588  itemtype=float,
589  default={},
590  )
591  sedboundaryterms = pexConfig.ConfigField(
592  doc="Mapping from bands to SED boundary term names used is sedterms.",
593  dtype=SedboundarytermDict,
594  )
595  sedterms = pexConfig.ConfigField(
596  doc="Mapping from terms to bands for fgcm linear SED approximations.",
597  dtype=SedtermDict,
598  )
599  sigFgcmMaxErr = pexConfig.Field(
600  doc="Maximum mag error for fitting sigma_FGCM",
601  dtype=float,
602  default=0.01,
603  )
604  sigFgcmMaxEGrayDict = pexConfig.DictField(
605  doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
606  "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
607  "should be 0.05."),
608  keytype=str,
609  itemtype=float,
610  default={},
611  )
612  ccdGrayMaxStarErr = pexConfig.Field(
613  doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
614  "computation"),
615  dtype=float,
616  default=0.10,
617  )
618  approxThroughputDict = pexConfig.DictField(
619  doc=("Per-band specification of the approximate overall throughput at the start of "
620  "calibration observations. Must have one entry per band. Typically should "
621  "be 1.0."),
622  keytype=str,
623  itemtype=float,
624  default={},
625  )
626  sigmaCalRange = pexConfig.ListField(
627  doc="Allowed range for systematic error floor estimation",
628  dtype=float,
629  default=(0.001, 0.003),
630  )
631  sigmaCalFitPercentile = pexConfig.ListField(
632  doc="Magnitude percentile range to fit systematic error floor",
633  dtype=float,
634  default=(0.05, 0.15),
635  )
636  sigmaCalPlotPercentile = pexConfig.ListField(
637  doc="Magnitude percentile range to plot systematic error floor",
638  dtype=float,
639  default=(0.05, 0.95),
640  )
641  sigma0Phot = pexConfig.Field(
642  doc="Systematic error floor for all zeropoints",
643  dtype=float,
644  default=0.003,
645  )
646  mapLongitudeRef = pexConfig.Field(
647  doc="Reference longitude for plotting maps",
648  dtype=float,
649  default=0.0,
650  )
651  mapNSide = pexConfig.Field(
652  doc="Healpix nside for plotting maps",
653  dtype=int,
654  default=256,
655  )
656  outfileBase = pexConfig.Field(
657  doc="Filename start for plot output files",
658  dtype=str,
659  default=None,
660  )
661  starColorCuts = pexConfig.ListField(
662  doc="Encoded star-color cuts (to be cleaned up)",
663  dtype=str,
664  default=("NO_DATA",),
665  )
666  colorSplitBands = pexConfig.ListField(
667  doc="Band names to use to split stars by color. Must have 2 entries.",
668  dtype=str,
669  length=2,
670  default=('g', 'i'),
671  )
672  modelMagErrors = pexConfig.Field(
673  doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
674  dtype=bool,
675  default=True,
676  )
677  useQuadraticPwv = pexConfig.Field(
678  doc="Model PWV with a quadratic term for variation through the night?",
679  dtype=bool,
680  default=False,
681  )
682  instrumentParsPerBand = pexConfig.Field(
683  doc=("Model instrumental parameters per band? "
684  "Otherwise, instrumental parameters (QE changes with time) are "
685  "shared among all bands."),
686  dtype=bool,
687  default=False,
688  )
689  instrumentSlopeMinDeltaT = pexConfig.Field(
690  doc=("Minimum time change (in days) between observations to use in constraining "
691  "instrument slope."),
692  dtype=float,
693  default=20.0,
694  )
695  fitMirrorChromaticity = pexConfig.Field(
696  doc="Fit (intraband) mirror chromatic term?",
697  dtype=bool,
698  default=False,
699  )
700  coatingMjds = pexConfig.ListField(
701  doc="Mirror coating dates in MJD",
702  dtype=float,
703  default=(0.0,),
704  )
705  outputStandardsBeforeFinalCycle = pexConfig.Field(
706  doc="Output standard stars prior to final cycle? Used in debugging.",
707  dtype=bool,
708  default=False,
709  )
710  outputZeropointsBeforeFinalCycle = pexConfig.Field(
711  doc="Output standard stars prior to final cycle? Used in debugging.",
712  dtype=bool,
713  default=False,
714  )
715  useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
716  doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
717  "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
718  keytype=str,
719  itemtype=bool,
720  default={},
721  )
722  autoPhotometricCutNSig = pexConfig.Field(
723  doc=("Number of sigma for automatic computation of (low) photometric cut. "
724  "Cut is based on exposure gray width (per band), unless "
725  "useRepeatabilityForExpGrayCuts is set, in which case the star "
726  "repeatability is used (also per band)."),
727  dtype=float,
728  default=3.0,
729  )
730  autoHighCutNSig = pexConfig.Field(
731  doc=("Number of sigma for automatic computation of (high) outlier cut. "
732  "Cut is based on exposure gray width (per band), unless "
733  "useRepeatabilityForExpGrayCuts is set, in which case the star "
734  "repeatability is used (also per band)."),
735  dtype=float,
736  default=4.0,
737  )
738  quietMode = pexConfig.Field(
739  doc="Be less verbose with logging.",
740  dtype=bool,
741  default=False,
742  )
743  doPlots = pexConfig.Field(
744  doc="Make fgcm QA plots.",
745  dtype=bool,
746  default=True,
747  )
748  randomSeed = pexConfig.Field(
749  doc="Random seed for fgcm for consistency in tests.",
750  dtype=int,
751  default=None,
752  optional=True,
753  )
754 
755  def validate(self):
756  super().validate()
757 
758  if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
759  msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
760  raise RuntimeError(msg)
761  if self.connections.cycleNumber != str(self.cycleNumber):
762  msg = "cycleNumber in template must be equal to connections.cycleNumber"
763  raise RuntimeError(msg)
764 
765  for band in self.fitBands:
766  if band not in self.bands:
767  msg = 'fitBand %s not in bands' % (band)
768  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
769  for band in self.requiredBands:
770  if band not in self.bands:
771  msg = 'requiredBand %s not in bands' % (band)
772  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
773  for band in self.colorSplitBands:
774  if band not in self.bands:
775  msg = 'colorSplitBand %s not in bands' % (band)
776  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
777  for band in self.bands:
778  if band not in self.superStarSubCcdDict:
779  msg = 'band %s not in superStarSubCcdDict' % (band)
780  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
781  self, msg)
782  if band not in self.ccdGraySubCcdDict:
783  msg = 'band %s not in ccdGraySubCcdDict' % (band)
784  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
785  self, msg)
786  if band not in self.expGrayPhotometricCutDict:
787  msg = 'band %s not in expGrayPhotometricCutDict' % (band)
788  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
789  self, msg)
790  if band not in self.expGrayHighCutDict:
791  msg = 'band %s not in expGrayHighCutDict' % (band)
792  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
793  self, msg)
794  if band not in self.expVarGrayPhotometricCutDict:
795  msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
796  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
797  self, msg)
798  if band not in self.sigFgcmMaxEGrayDict:
799  msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
800  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
801  self, msg)
802  if band not in self.approxThroughputDict:
803  msg = 'band %s not in approxThroughputDict' % (band)
804  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
805  self, msg)
806  if band not in self.useRepeatabilityForExpGrayCutsDict:
807  msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
808  raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
809  self, msg)
810 
811 
812 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
813  """Subclass of TaskRunner for fgcmFitCycleTask
814 
815  fgcmFitCycleTask.run() takes one argument, the butler, and uses
816  stars and visits previously extracted from dataRefs by
817  fgcmBuildStars.
818  This Runner does not perform any dataRef parallelization, but the FGCM
819  code called by the Task uses python multiprocessing (see the "ncores"
820  config option).
821  """
822 
823  @staticmethod
824  def getTargetList(parsedCmd):
825  """
826  Return a list with one element, the butler.
827  """
828  return [parsedCmd.butler]
829 
830  def __call__(self, butler):
831  """
832  Parameters
833  ----------
834  butler: `lsst.daf.persistence.Butler`
835 
836  Returns
837  -------
838  exitStatus: `list` with `pipeBase.Struct`
839  exitStatus (0: success; 1: failure)
840  """
841 
842  task = self.TaskClass(config=self.config, log=self.log)
843 
844  exitStatus = 0
845  if self.doRaise:
846  task.runDataRef(butler)
847  else:
848  try:
849  task.runDataRef(butler)
850  except Exception as e:
851  exitStatus = 1
852  task.log.fatal("Failed: %s" % e)
853  if not isinstance(e, pipeBase.TaskError):
854  traceback.print_exc(file=sys.stderr)
855 
856  task.writeMetadata(butler)
857 
858  # The task does not return any results:
859  return [pipeBase.Struct(exitStatus=exitStatus)]
860 
861  def run(self, parsedCmd):
862  """
863  Run the task, with no multiprocessing
864 
865  Parameters
866  ----------
867  parsedCmd: ArgumentParser parsed command line
868  """
869 
870  resultList = []
871 
872  if self.precall(parsedCmd):
873  targetList = self.getTargetList(parsedCmd)
874  # make sure that we only get 1
875  resultList = self(targetList[0])
876 
877  return resultList
878 
879 
880 class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
881  """
882  Run Single fit cycle for FGCM global calibration
883  """
884 
885  ConfigClass = FgcmFitCycleConfig
886  RunnerClass = FgcmFitCycleRunner
887  _DefaultName = "fgcmFitCycle"
888 
889  def __init__(self, butler=None, initInputs=None, **kwargs):
890  super().__init__(**kwargs)
891 
892  # no saving of metadata for now
893  def _getMetadataName(self):
894  return None
895 
896  def runQuantum(self, butlerQC, inputRefs, outputRefs):
897  camera = butlerQC.get(inputRefs.camera)
898 
899  dataRefDict = {}
900 
901  dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
902  dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
903  dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
904  dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
905  dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
906  if self.config.doReferenceCalibration:
907  dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
908  if self.config.cycleNumber > 0:
909  dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
910  dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
911 
912  fgcmDatasetDict = None
913  if self.config.doMultipleCycles:
914  # Run multiple cycles at once.
915  config = copy.copy(self.config)
916  config.update(cycleNumber=0)
917  for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
918  if cycle == self.config.multipleCyclesFinalCycleNumber:
919  config.update(isFinalCycle=True)
920 
921  if cycle > 0:
922  dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
923  dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
924 
925  fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
926  butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
927  getattr(outputRefs, f'fgcmFitParameters{cycle}'))
928  butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
929  getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
930  if self.outputZeropoints:
931  butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
932  getattr(outputRefs, f'fgcmZeropoints{cycle}'))
933  butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
934  getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
935  if self.outputStandards:
936  butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
937  getattr(outputRefs, f'fgcmStandardStars{cycle}'))
938  else:
939  # Run a single cycle
940  fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
941 
942  butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
943  butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
944  if self.outputZeropoints:
945  butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
946  butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
947  if self.outputStandards:
948  butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
949 
950  @timeMethod
951  def runDataRef(self, butler):
952  """
953  Run a single fit cycle for FGCM
954 
955  Parameters
956  ----------
957  butler: `lsst.daf.persistence.Butler`
958  """
959  self._checkDatasetsExist(butler)
960 
961  dataRefDict = {}
962  dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
963  dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
964  dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
965  dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
966  dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
967  if self.config.doReferenceCalibration:
968  dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
969  if self.config.cycleNumber > 0:
970  lastCycle = self.config.cycleNumber - 1
971  dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
972  fgcmcycle=lastCycle)
973  dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
974  fgcmcycle=lastCycle)
975 
976  camera = butler.get('camera')
977  fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
978 
979  butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
980  fgcmcycle=self.config.cycleNumber)
981  butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
982  fgcmcycle=self.config.cycleNumber)
983  if self.outputZeropoints:
984  butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
985  fgcmcycle=self.config.cycleNumber)
986  butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
987  fgcmcycle=self.config.cycleNumber)
988  if self.outputStandards:
989  butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
990  fgcmcycle=self.config.cycleNumber)
991 
992  def writeConfig(self, butler, clobber=False, doBackup=True):
993  """Write the configuration used for processing the data, or check that an existing
994  one is equal to the new one if present. This is an override of the regular
995  version from pipe_base that knows about fgcmcycle.
996 
997  Parameters
998  ----------
999  butler : `lsst.daf.persistence.Butler`
1000  Data butler used to write the config. The config is written to dataset type
1001  `CmdLineTask._getConfigName`.
1002  clobber : `bool`, optional
1003  A boolean flag that controls what happens if a config already has been saved:
1004  - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1005  - `False`: raise `TaskError` if this config does not match the existing config.
1006  doBackup : `bool`, optional
1007  Set to `True` to backup the config files if clobbering.
1008  """
1009  configName = self._getConfigName()
1010  if configName is None:
1011  return
1012  if clobber:
1013  butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1014  elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1015  # this may be subject to a race condition; see #2789
1016  try:
1017  oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1018  except Exception as exc:
1019  raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1020  (configName, exc))
1021 
1022  def logConfigMismatch(msg):
1023  self.log.fatal("Comparing configuration: %s", msg)
1024 
1025  if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1026  raise pipeBase.TaskError(
1027  f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1028  " must be consistent within the same output repo (override with --clobber-config)")
1029  else:
1030  butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1031 
1032  def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1033  """
1034  Run the fit cycle
1035 
1036  Parameters
1037  ----------
1038  camera : `lsst.afw.cameraGeom.Camera`
1039  dataRefDict : `dict`
1040  All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1041  `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1042  dataRef dictionary with keys:
1043 
1044  ``"fgcmLookUpTable"``
1045  dataRef for the FGCM look-up table.
1046  ``"fgcmVisitCatalog"``
1047  dataRef for visit summary catalog.
1048  ``"fgcmStarObservations"``
1049  dataRef for star observation catalog.
1050  ``"fgcmStarIds"``
1051  dataRef for star id catalog.
1052  ``"fgcmStarIndices"``
1053  dataRef for star index catalog.
1054  ``"fgcmReferenceStars"``
1055  dataRef for matched reference star catalog.
1056  ``"fgcmFlaggedStars"``
1057  dataRef for flagged star catalog.
1058  ``"fgcmFitParameters"``
1059  dataRef for fit parameter catalog.
1060  config : `lsst.pex.config.Config`, optional
1061  Configuration to use to override self.config.
1062 
1063  Returns
1064  -------
1065  fgcmDatasetDict : `dict`
1066  Dictionary of datasets to persist.
1067  """
1068  if config is not None:
1069  _config = config
1070  else:
1071  _config = self.config
1072 
1073  # Set defaults on whether to output standards and zeropoints
1074  self.maxIter = _config.maxIterBeforeFinalCycle
1075  self.outputStandards = _config.outputStandardsBeforeFinalCycle
1076  self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1077  self.resetFitParameters = True
1078 
1079  if _config.isFinalCycle:
1080  # This is the final fit cycle, so we do not want to reset fit
1081  # parameters, we want to run a final "clean-up" with 0 fit iterations,
1082  # and we always want to output standards and zeropoints
1083  self.maxIter = 0
1084  self.outputStandards = True
1085  self.outputZeropoints = True
1086  self.resetFitParameters = False
1087 
1088  lutCat = dataRefDict['fgcmLookUpTable'].get()
1089  fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1090  dict(_config.physicalFilterMap))
1091  del lutCat
1092 
1093  configDict = makeConfigDict(_config, self.log, camera,
1094  self.maxIter, self.resetFitParameters,
1095  self.outputZeropoints,
1096  lutIndexVals[0]['FILTERNAMES'])
1097 
1098  # next we need the exposure/visit information
1099  visitCat = dataRefDict['fgcmVisitCatalog'].get()
1100  fgcmExpInfo = translateVisitCatalog(visitCat)
1101  del visitCat
1102 
1103  # Use the first orientation.
1104  # TODO: DM-21215 will generalize to arbitrary camera orientations
1105  ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1106 
1107  noFitsDict = {'lutIndex': lutIndexVals,
1108  'lutStd': lutStd,
1109  'expInfo': fgcmExpInfo,
1110  'ccdOffsets': ccdOffsets}
1111 
1112  # set up the fitter object
1113  fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1114  noFitsDict=noFitsDict, noOutput=True)
1115 
1116  # create the parameter object
1117  if (fgcmFitCycle.initialCycle):
1118  # cycle = 0, initial cycle
1119  fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1120  fgcmLut,
1121  fgcmExpInfo)
1122  else:
1123  if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog):
1124  parCat = dataRefDict['fgcmFitParameters']
1125  else:
1126  parCat = dataRefDict['fgcmFitParameters'].get()
1127  inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1128  del parCat
1129  fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1130  fgcmExpInfo,
1131  inParInfo,
1132  inParams,
1133  inSuperStar)
1134 
1135  # set up the stars...
1136  fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1137 
1138  starObs = dataRefDict['fgcmStarObservations'].get()
1139  starIds = dataRefDict['fgcmStarIds'].get()
1140  starIndices = dataRefDict['fgcmStarIndices'].get()
1141 
1142  # grab the flagged stars if available
1143  if 'fgcmFlaggedStars' in dataRefDict:
1144  if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1145  flaggedStars = dataRefDict['fgcmFlaggedStars']
1146  else:
1147  flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1148  flagId = flaggedStars['objId'][:]
1149  flagFlag = flaggedStars['objFlag'][:]
1150  else:
1151  flaggedStars = None
1152  flagId = None
1153  flagFlag = None
1154 
1155  if _config.doReferenceCalibration:
1156  refStars = dataRefDict['fgcmReferenceStars'].get()
1157 
1158  refMag, refMagErr = extractReferenceMags(refStars,
1159  _config.bands,
1160  _config.physicalFilterMap)
1161  refId = refStars['fgcm_id'][:]
1162  else:
1163  refStars = None
1164  refId = None
1165  refMag = None
1166  refMagErr = None
1167 
1168  # match star observations to visits
1169  # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1170  # actually be transferred into fgcm using the indexing below.
1171  visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1172 
1173  # The fgcmStars.loadStars method will copy all the star information into
1174  # special shared memory objects that will not blow up the memory usage when
1175  # used with python multiprocessing. Once all the numbers are copied,
1176  # it is necessary to release all references to the objects that previously
1177  # stored the data to ensure that the garbage collector can clear the memory,
1178  # and ensure that this memory is not copied when multiprocessing kicks in.
1179 
1180  # We determine the conversion from the native units (typically radians) to
1181  # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1182  # numpy arrays rather than Angles, which would we approximately 600x slower.
1183  conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1184 
1185  fgcmStars.loadStars(fgcmPars,
1186  starObs['visit'][starIndices['obsIndex']],
1187  starObs['ccd'][starIndices['obsIndex']],
1188  starObs['ra'][starIndices['obsIndex']] * conv,
1189  starObs['dec'][starIndices['obsIndex']] * conv,
1190  starObs['instMag'][starIndices['obsIndex']],
1191  starObs['instMagErr'][starIndices['obsIndex']],
1192  fgcmExpInfo['FILTERNAME'][visitIndex],
1193  starIds['fgcm_id'][:],
1194  starIds['ra'][:],
1195  starIds['dec'][:],
1196  starIds['obsArrIndex'][:],
1197  starIds['nObs'][:],
1198  obsX=starObs['x'][starIndices['obsIndex']],
1199  obsY=starObs['y'][starIndices['obsIndex']],
1200  obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1201  psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1202  refID=refId,
1203  refMag=refMag,
1204  refMagErr=refMagErr,
1205  flagID=flagId,
1206  flagFlag=flagFlag,
1207  computeNobs=True)
1208 
1209  # Release all references to temporary objects holding star data (see above)
1210  del starObs
1211  del starIds
1212  del starIndices
1213  del flagId
1214  del flagFlag
1215  del flaggedStars
1216  del refStars
1217  del refId
1218  del refMag
1219  del refMagErr
1220 
1221  # and set the bits in the cycle object
1222  fgcmFitCycle.setLUT(fgcmLut)
1223  fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1224  fgcmFitCycle.setPars(fgcmPars)
1225 
1226  # finish the setup
1227  fgcmFitCycle.finishSetup()
1228 
1229  # and run
1230  fgcmFitCycle.run()
1231 
1232 
1235 
1236  fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1237 
1238  # Output the config for the next cycle
1239  # We need to make a copy since the input one has been frozen
1240 
1241  updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1242  i, b in enumerate(_config.bands)}
1243  updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1244  i, band in enumerate(_config.bands)}
1245 
1246  outConfig = copy.copy(_config)
1247  outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1248  precomputeSuperStarInitialCycle=False,
1249  freezeStdAtmosphere=False,
1250  expGrayPhotometricCutDict=updatedPhotometricCutDict,
1251  expGrayHighCutDict=updatedHighCutDict)
1252 
1253  outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1254  cycleNumber=str(_config.cycleNumber + 1))
1255 
1256  configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1257  outConfig.cycleNumber)
1258  outConfig.save(configFileName)
1259 
1260  if _config.isFinalCycle == 1:
1261  # We are done, ready to output products
1262  self.log.info("Everything is in place to run fgcmOutputProducts.py")
1263  else:
1264  self.log.info("Saved config for next cycle to %s" % (configFileName))
1265  self.log.info("Be sure to look at:")
1266  self.log.info(" config.expGrayPhotometricCut")
1267  self.log.info(" config.expGrayHighCut")
1268  self.log.info("If you are satisfied with the fit, please set:")
1269  self.log.info(" config.isFinalCycle = True")
1270 
1271  fgcmFitCycle.freeSharedMemory()
1272 
1273  return fgcmDatasetDict, outConfig
1274 
1275  def _checkDatasetsExist(self, butler):
1276  """
1277  Check if necessary datasets exist to run fgcmFitCycle
1278 
1279  Parameters
1280  ----------
1281  butler: `lsst.daf.persistence.Butler`
1282 
1283  Raises
1284  ------
1285  RuntimeError
1286  If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1287  fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1288  If cycleNumber > 0, then also checks for fgcmFitParameters,
1289  fgcmFlaggedStars.
1290  """
1291 
1292  if not butler.datasetExists('fgcmVisitCatalog'):
1293  raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1294  if not butler.datasetExists('fgcmStarObservations'):
1295  raise RuntimeError("Could not find fgcmStarObservations in repo!")
1296  if not butler.datasetExists('fgcmStarIds'):
1297  raise RuntimeError("Could not find fgcmStarIds in repo!")
1298  if not butler.datasetExists('fgcmStarIndices'):
1299  raise RuntimeError("Could not find fgcmStarIndices in repo!")
1300  if not butler.datasetExists('fgcmLookUpTable'):
1301  raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1302 
1303  # Need additional datasets if we are not the initial cycle
1304  if (self.config.cycleNumber > 0):
1305  if not butler.datasetExists('fgcmFitParameters',
1306  fgcmcycle=self.config.cycleNumber-1):
1307  raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1308  (self.config.cycleNumber-1))
1309  if not butler.datasetExists('fgcmFlaggedStars',
1310  fgcmcycle=self.config.cycleNumber-1):
1311  raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1312  (self.config.cycleNumber-1))
1313 
1314  # And additional dataset if we want reference calibration
1315  if self.config.doReferenceCalibration:
1316  if not butler.datasetExists('fgcmReferenceStars'):
1317  raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1318  "doReferenceCalibration is True.")
1319 
1320  def _loadParameters(self, parCat):
1321  """
1322  Load FGCM parameters from a previous fit cycle
1323 
1324  Parameters
1325  ----------
1326  parCat : `lsst.afw.table.BaseCatalog`
1327  Parameter catalog in afw table form.
1328 
1329  Returns
1330  -------
1331  inParInfo: `numpy.ndarray`
1332  Numpy array parameter information formatted for input to fgcm
1333  inParameters: `numpy.ndarray`
1334  Numpy array parameter values formatted for input to fgcm
1335  inSuperStar: `numpy.array`
1336  Superstar flat formatted for input to fgcm
1337  """
1338  parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1339  parFitBands = np.array(parCat[0]['fitBands'].split(','))
1340 
1341  inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1342  ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1343  (parLutFilterNames.size, )),
1344  ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1345  ('LNTAUUNIT', 'f8'),
1346  ('LNTAUSLOPEUNIT', 'f8'),
1347  ('ALPHAUNIT', 'f8'),
1348  ('LNPWVUNIT', 'f8'),
1349  ('LNPWVSLOPEUNIT', 'f8'),
1350  ('LNPWVQUADRATICUNIT', 'f8'),
1351  ('LNPWVGLOBALUNIT', 'f8'),
1352  ('O3UNIT', 'f8'),
1353  ('QESYSUNIT', 'f8'),
1354  ('FILTEROFFSETUNIT', 'f8'),
1355  ('HASEXTERNALPWV', 'i2'),
1356  ('HASEXTERNALTAU', 'i2')])
1357  inParInfo['NCCD'] = parCat['nCcd']
1358  inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1359  inParInfo['FITBANDS'][:] = parFitBands
1360  inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1361  inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1362 
1363  inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1364  ('PARO3', 'f8', (parCat['parO3'].size, )),
1365  ('PARLNTAUINTERCEPT', 'f8',
1366  (parCat['parLnTauIntercept'].size, )),
1367  ('PARLNTAUSLOPE', 'f8',
1368  (parCat['parLnTauSlope'].size, )),
1369  ('PARLNPWVINTERCEPT', 'f8',
1370  (parCat['parLnPwvIntercept'].size, )),
1371  ('PARLNPWVSLOPE', 'f8',
1372  (parCat['parLnPwvSlope'].size, )),
1373  ('PARLNPWVQUADRATIC', 'f8',
1374  (parCat['parLnPwvQuadratic'].size, )),
1375  ('PARQESYSINTERCEPT', 'f8',
1376  (parCat['parQeSysIntercept'].size, )),
1377  ('COMPQESYSSLOPE', 'f8',
1378  (parCat['compQeSysSlope'].size, )),
1379  ('PARFILTEROFFSET', 'f8',
1380  (parCat['parFilterOffset'].size, )),
1381  ('PARFILTEROFFSETFITFLAG', 'i2',
1382  (parCat['parFilterOffsetFitFlag'].size, )),
1383  ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1384  ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1385  ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1386  (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1387  ('COMPABSTHROUGHPUT', 'f8',
1388  (parCat['compAbsThroughput'].size, )),
1389  ('COMPREFOFFSET', 'f8',
1390  (parCat['compRefOffset'].size, )),
1391  ('COMPREFSIGMA', 'f8',
1392  (parCat['compRefSigma'].size, )),
1393  ('COMPMIRRORCHROMATICITY', 'f8',
1394  (parCat['compMirrorChromaticity'].size, )),
1395  ('MIRRORCHROMATICITYPIVOT', 'f8',
1396  (parCat['mirrorChromaticityPivot'].size, )),
1397  ('COMPMEDIANSEDSLOPE', 'f8',
1398  (parCat['compMedianSedSlope'].size, )),
1399  ('COMPAPERCORRPIVOT', 'f8',
1400  (parCat['compAperCorrPivot'].size, )),
1401  ('COMPAPERCORRSLOPE', 'f8',
1402  (parCat['compAperCorrSlope'].size, )),
1403  ('COMPAPERCORRSLOPEERR', 'f8',
1404  (parCat['compAperCorrSlopeErr'].size, )),
1405  ('COMPAPERCORRRANGE', 'f8',
1406  (parCat['compAperCorrRange'].size, )),
1407  ('COMPMODELERREXPTIMEPIVOT', 'f8',
1408  (parCat['compModelErrExptimePivot'].size, )),
1409  ('COMPMODELERRFWHMPIVOT', 'f8',
1410  (parCat['compModelErrFwhmPivot'].size, )),
1411  ('COMPMODELERRSKYPIVOT', 'f8',
1412  (parCat['compModelErrSkyPivot'].size, )),
1413  ('COMPMODELERRPARS', 'f8',
1414  (parCat['compModelErrPars'].size, )),
1415  ('COMPEXPGRAY', 'f8',
1416  (parCat['compExpGray'].size, )),
1417  ('COMPVARGRAY', 'f8',
1418  (parCat['compVarGray'].size, )),
1419  ('COMPEXPDELTAMAGBKG', 'f8',
1420  (parCat['compExpDeltaMagBkg'].size, )),
1421  ('COMPNGOODSTARPEREXP', 'i4',
1422  (parCat['compNGoodStarPerExp'].size, )),
1423  ('COMPSIGFGCM', 'f8',
1424  (parCat['compSigFgcm'].size, )),
1425  ('COMPSIGMACAL', 'f8',
1426  (parCat['compSigmaCal'].size, )),
1427  ('COMPRETRIEVEDLNPWV', 'f8',
1428  (parCat['compRetrievedLnPwv'].size, )),
1429  ('COMPRETRIEVEDLNPWVRAW', 'f8',
1430  (parCat['compRetrievedLnPwvRaw'].size, )),
1431  ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1432  (parCat['compRetrievedLnPwvFlag'].size, )),
1433  ('COMPRETRIEVEDTAUNIGHT', 'f8',
1434  (parCat['compRetrievedTauNight'].size, ))])
1435 
1436  inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1437  inParams['PARO3'][:] = parCat['parO3'][0, :]
1438  inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1439  inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1440  inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1441  inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1442  inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1443  inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1444  inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1445  inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1446  inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1447  inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1448  inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1449  inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1450  inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1451  inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1452  inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1453  inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1454  inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1455  inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1456  inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1457  inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1458  inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1459  inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1460  inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1461  inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1462  inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1463  inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1464  inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1465  inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1466  inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1467  inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1468  inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1469  inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1470  inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1471  inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1472  inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1473  inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1474 
1475  inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1476  inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1477 
1478  return (inParInfo, inParams, inSuperStar)
1479 
1480  def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1481  """
1482  Persist FGCM datasets through the butler.
1483 
1484  Parameters
1485  ----------
1486  fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1487  Fgcm Fit cycle object
1488  """
1489  fgcmDatasetDict = {}
1490 
1491  # Save the parameters
1492  parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1493 
1494  parSchema = afwTable.Schema()
1495 
1496  comma = ','
1497  lutFilterNameString = comma.join([n.decode('utf-8')
1498  for n in parInfo['LUTFILTERNAMES'][0]])
1499  fitBandString = comma.join([n.decode('utf-8')
1500  for n in parInfo['FITBANDS'][0]])
1501 
1502  parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1503  lutFilterNameString, fitBandString)
1504  parCat = self._makeParCatalog(parSchema, parInfo, pars,
1505  fgcmFitCycle.fgcmPars.parSuperStarFlat,
1506  lutFilterNameString, fitBandString)
1507 
1508  fgcmDatasetDict['fgcmFitParameters'] = parCat
1509 
1510  # Save the indices of the flagged stars
1511  # (stars that have been (a) reserved from the fit for testing and
1512  # (b) bad stars that have failed quality checks.)
1513  flagStarSchema = self._makeFlagStarSchema()
1514  flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1515  flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1516 
1517  fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1518 
1519  # Save the zeropoint information and atmospheres only if desired
1520  if self.outputZeropoints:
1521  superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1522  zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1523 
1524  zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1525  zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1526 
1527  fgcmDatasetDict['fgcmZeropoints'] = zptCat
1528 
1529  # Save atmosphere values
1530  # These are generated by the same code that generates zeropoints
1531  atmSchema = makeAtmSchema()
1532  atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1533 
1534  fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1535 
1536  # Save the standard stars (if configured)
1537  if self.outputStandards:
1538  stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1539  stdSchema = makeStdSchema(len(goodBands))
1540  stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1541 
1542  fgcmDatasetDict['fgcmStandardStars'] = stdCat
1543 
1544  return fgcmDatasetDict
1545 
1546  def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1547  lutFilterNameString, fitBandString):
1548  """
1549  Make the parameter persistence schema
1550 
1551  Parameters
1552  ----------
1553  parInfo: `numpy.ndarray`
1554  Parameter information returned by fgcm
1555  pars: `numpy.ndarray`
1556  Parameter values returned by fgcm
1557  parSuperStarFlat: `numpy.array`
1558  Superstar flat values returned by fgcm
1559  lutFilterNameString: `str`
1560  Combined string of all the lutFilterNames
1561  fitBandString: `str`
1562  Combined string of all the fitBands
1563 
1564  Returns
1565  -------
1566  parSchema: `afwTable.schema`
1567  """
1568 
1569  parSchema = afwTable.Schema()
1570 
1571  # parameter info section
1572  parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1573  parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1574  size=len(lutFilterNameString))
1575  parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1576  size=len(fitBandString))
1577  parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1578  parSchema.addField('lnTauSlopeUnit', type=np.float64,
1579  doc='Step units for ln(AOD) slope')
1580  parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1581  parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1582  parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1583  doc='Step units for ln(pwv) slope')
1584  parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1585  doc='Step units for ln(pwv) quadratic term')
1586  parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1587  doc='Step units for global ln(pwv) parameters')
1588  parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1589  parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1590  parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1591  parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1592  parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1593 
1594  # parameter section
1595  parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1596  size=pars['PARALPHA'].size)
1597  parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1598  size=pars['PARO3'].size)
1599  parSchema.addField('parLnTauIntercept', type='ArrayD',
1600  doc='ln(Tau) intercept parameter vector',
1601  size=pars['PARLNTAUINTERCEPT'].size)
1602  parSchema.addField('parLnTauSlope', type='ArrayD',
1603  doc='ln(Tau) slope parameter vector',
1604  size=pars['PARLNTAUSLOPE'].size)
1605  parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1606  size=pars['PARLNPWVINTERCEPT'].size)
1607  parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1608  size=pars['PARLNPWVSLOPE'].size)
1609  parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1610  size=pars['PARLNPWVQUADRATIC'].size)
1611  parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1612  size=pars['PARQESYSINTERCEPT'].size)
1613  parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1614  size=pars[0]['COMPQESYSSLOPE'].size)
1615  parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1616  size=pars['PARFILTEROFFSET'].size)
1617  parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1618  size=pars['PARFILTEROFFSETFITFLAG'].size)
1619  parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1620  doc='Global scale for retrieved ln(pwv)')
1621  parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1622  doc='Global offset for retrieved ln(pwv)')
1623  parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1624  doc='Nightly offset for retrieved ln(pwv)',
1625  size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1626  parSchema.addField('compAbsThroughput', type='ArrayD',
1627  doc='Absolute throughput (relative to transmission curves)',
1628  size=pars['COMPABSTHROUGHPUT'].size)
1629  parSchema.addField('compRefOffset', type='ArrayD',
1630  doc='Offset between reference stars and calibrated stars',
1631  size=pars['COMPREFOFFSET'].size)
1632  parSchema.addField('compRefSigma', type='ArrayD',
1633  doc='Width of reference star/calibrated star distribution',
1634  size=pars['COMPREFSIGMA'].size)
1635  parSchema.addField('compMirrorChromaticity', type='ArrayD',
1636  doc='Computed mirror chromaticity terms',
1637  size=pars['COMPMIRRORCHROMATICITY'].size)
1638  parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1639  doc='Mirror chromaticity pivot mjd',
1640  size=pars['MIRRORCHROMATICITYPIVOT'].size)
1641  parSchema.addField('compMedianSedSlope', type='ArrayD',
1642  doc='Computed median SED slope (per band)',
1643  size=pars['COMPMEDIANSEDSLOPE'].size)
1644  parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1645  size=pars['COMPAPERCORRPIVOT'].size)
1646  parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1647  size=pars['COMPAPERCORRSLOPE'].size)
1648  parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1649  size=pars['COMPAPERCORRSLOPEERR'].size)
1650  parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1651  size=pars['COMPAPERCORRRANGE'].size)
1652  parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1653  size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1654  parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1655  size=pars['COMPMODELERRFWHMPIVOT'].size)
1656  parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1657  size=pars['COMPMODELERRSKYPIVOT'].size)
1658  parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1659  size=pars['COMPMODELERRPARS'].size)
1660  parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1661  size=pars['COMPEXPGRAY'].size)
1662  parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1663  size=pars['COMPVARGRAY'].size)
1664  parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1665  doc='Computed exposure offset due to background',
1666  size=pars['COMPEXPDELTAMAGBKG'].size)
1667  parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1668  doc='Computed number of good stars per exposure',
1669  size=pars['COMPNGOODSTARPEREXP'].size)
1670  parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1671  size=pars['COMPSIGFGCM'].size)
1672  parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1673  size=pars['COMPSIGMACAL'].size)
1674  parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1675  size=pars['COMPRETRIEVEDLNPWV'].size)
1676  parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1677  size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1678  parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1679  size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1680  parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1681  size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1682  # superstarflat section
1683  parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1684  size=4)
1685  parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1686  size=parSuperStarFlat.size)
1687 
1688  return parSchema
1689 
1690  def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1691  lutFilterNameString, fitBandString):
1692  """
1693  Make the FGCM parameter catalog for persistence
1694 
1695  Parameters
1696  ----------
1697  parSchema: `lsst.afw.table.Schema`
1698  Parameter catalog schema
1699  pars: `numpy.ndarray`
1700  FGCM parameters to put into parCat
1701  parSuperStarFlat: `numpy.array`
1702  FGCM superstar flat array to put into parCat
1703  lutFilterNameString: `str`
1704  Combined string of all the lutFilterNames
1705  fitBandString: `str`
1706  Combined string of all the fitBands
1707 
1708  Returns
1709  -------
1710  parCat: `afwTable.BasicCatalog`
1711  Atmosphere and instrumental model parameter catalog for persistence
1712  """
1713 
1714  parCat = afwTable.BaseCatalog(parSchema)
1715  parCat.reserve(1)
1716 
1717  # The parameter catalog just has one row, with many columns for all the
1718  # atmosphere and instrument fit parameters
1719  rec = parCat.addNew()
1720 
1721  # info section
1722  rec['nCcd'] = parInfo['NCCD']
1723  rec['lutFilterNames'] = lutFilterNameString
1724  rec['fitBands'] = fitBandString
1725  # note these are not currently supported here.
1726  rec['hasExternalPwv'] = 0
1727  rec['hasExternalTau'] = 0
1728 
1729  # parameter section
1730 
1731  scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1732 
1733  arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1734  'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1735  'parQeSysIntercept', 'compQeSysSlope',
1736  'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1737  'parFilterOffset', 'parFilterOffsetFitFlag',
1738  'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1739  'compMirrorChromaticity', 'mirrorChromaticityPivot',
1740  'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1741  'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1742  'compModelErrSkyPivot', 'compModelErrPars',
1743  'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1744  'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1745  'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1746  'compRetrievedTauNight']
1747 
1748  for scalarName in scalarNames:
1749  rec[scalarName] = pars[scalarName.upper()]
1750 
1751  for arrName in arrNames:
1752  rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1753 
1754  # superstar section
1755  rec['superstarSize'][:] = parSuperStarFlat.shape
1756  rec['superstar'][:] = parSuperStarFlat.flatten()
1757 
1758  return parCat
1759 
1760  def _makeFlagStarSchema(self):
1761  """
1762  Make the flagged-stars schema
1763 
1764  Returns
1765  -------
1766  flagStarSchema: `lsst.afw.table.Schema`
1767  """
1768 
1769  flagStarSchema = afwTable.Schema()
1770 
1771  flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1772  flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1773 
1774  return flagStarSchema
1775 
1776  def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1777  """
1778  Make the flagged star catalog for persistence
1779 
1780  Parameters
1781  ----------
1782  flagStarSchema: `lsst.afw.table.Schema`
1783  Flagged star schema
1784  flagStarStruct: `numpy.ndarray`
1785  Flagged star structure from fgcm
1786 
1787  Returns
1788  -------
1789  flagStarCat: `lsst.afw.table.BaseCatalog`
1790  Flagged star catalog for persistence
1791  """
1792 
1793  flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1794  flagStarCat.resize(flagStarStruct.size)
1795 
1796  flagStarCat['objId'][:] = flagStarStruct['OBJID']
1797  flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1798 
1799  return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:883
def makeStdSchema(nBands)
Definition: utilities.py:738
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:705
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:51
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:218
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:626
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:770
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:540
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:399
def translateVisitCatalog(visitCat)
Definition: utilities.py:348