23 """Perform a single fit cycle of FGCM.
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
42 import lsst.pex.config
as pexConfig
43 import lsst.pipe.base
as pipeBase
44 from lsst.pipe.base
import connectionTypes
45 import lsst.afw.table
as afwTable
46 from lsst.utils.timer
import timeMethod
48 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
49 from .utilities
import extractReferenceMags
50 from .utilities
import computeCcdOffsets, makeZptSchema, makeZptCat
51 from .utilities
import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
52 from .sedterms
import SedboundarytermDict, SedtermDict
53 from .utilities
import lookupStaticCalibrations
57 __all__ = [
'FgcmFitCycleConfig',
'FgcmFitCycleTask',
'FgcmFitCycleRunner']
59 MULTIPLE_CYCLES_MAX = 10
63 dimensions=(
"instrument",),
64 defaultTemplates={
"previousCycleNumber":
"-1",
66 camera = connectionTypes.PrerequisiteInput(
67 doc=
"Camera instrument",
69 storageClass=
"Camera",
70 dimensions=(
"instrument",),
71 lookupFunction=lookupStaticCalibrations,
75 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
76 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
77 "chromatic corrections."),
78 name=
"fgcmLookUpTable",
79 storageClass=
"Catalog",
80 dimensions=(
"instrument",),
84 fgcmVisitCatalog = connectionTypes.Input(
85 doc=
"Catalog of visit information for fgcm",
86 name=
"fgcmVisitCatalog",
87 storageClass=
"Catalog",
88 dimensions=(
"instrument",),
92 fgcmStarObservations = connectionTypes.Input(
93 doc=
"Catalog of star observations for fgcm",
94 name=
"fgcmStarObservations",
95 storageClass=
"Catalog",
96 dimensions=(
"instrument",),
100 fgcmStarIds = connectionTypes.Input(
101 doc=
"Catalog of fgcm calibration star IDs",
103 storageClass=
"Catalog",
104 dimensions=(
"instrument",),
108 fgcmStarIndices = connectionTypes.Input(
109 doc=
"Catalog of fgcm calibration star indices",
110 name=
"fgcmStarIndices",
111 storageClass=
"Catalog",
112 dimensions=(
"instrument",),
116 fgcmReferenceStars = connectionTypes.Input(
117 doc=
"Catalog of fgcm-matched reference stars",
118 name=
"fgcmReferenceStars",
119 storageClass=
"Catalog",
120 dimensions=(
"instrument",),
124 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
125 doc=
"Catalog of flagged stars for fgcm calibration from previous fit cycle",
126 name=
"fgcmFlaggedStars{previousCycleNumber}",
127 storageClass=
"Catalog",
128 dimensions=(
"instrument",),
132 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
133 doc=
"Catalog of fgcm fit parameters from previous fit cycle",
134 name=
"fgcmFitParameters{previousCycleNumber}",
135 storageClass=
"Catalog",
136 dimensions=(
"instrument",),
140 fgcmFitParameters = connectionTypes.Output(
141 doc=
"Catalog of fgcm fit parameters from current fit cycle",
142 name=
"fgcmFitParameters{cycleNumber}",
143 storageClass=
"Catalog",
144 dimensions=(
"instrument",),
147 fgcmFlaggedStars = connectionTypes.Output(
148 doc=
"Catalog of flagged stars for fgcm calibration from current fit cycle",
149 name=
"fgcmFlaggedStars{cycleNumber}",
150 storageClass=
"Catalog",
151 dimensions=(
"instrument",),
154 fgcmZeropoints = connectionTypes.Output(
155 doc=
"Catalog of fgcm zeropoint data from current fit cycle",
156 name=
"fgcmZeropoints{cycleNumber}",
157 storageClass=
"Catalog",
158 dimensions=(
"instrument",),
161 fgcmAtmosphereParameters = connectionTypes.Output(
162 doc=
"Catalog of atmospheric fit parameters from current fit cycle",
163 name=
"fgcmAtmosphereParameters{cycleNumber}",
164 storageClass=
"Catalog",
165 dimensions=(
"instrument",),
168 fgcmStandardStars = connectionTypes.Output(
169 doc=
"Catalog of standard star magnitudes from current fit cycle",
170 name=
"fgcmStandardStars{cycleNumber}",
171 storageClass=
"SimpleCatalog",
172 dimensions=(
"instrument",),
178 for cycle
in range(MULTIPLE_CYCLES_MAX):
179 vars()[f
"fgcmFitParameters{cycle}"] = connectionTypes.Output(
180 doc=f
"Catalog of fgcm fit parameters from fit cycle {cycle}",
181 name=f
"fgcmFitParameters{cycle}",
182 storageClass=
"Catalog",
183 dimensions=(
"instrument",),
185 vars()[f
"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
186 doc=f
"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
187 name=f
"fgcmFlaggedStars{cycle}",
188 storageClass=
"Catalog",
189 dimensions=(
"instrument",),
191 vars()[f
"fgcmZeropoints{cycle}"] = connectionTypes.Output(
192 doc=f
"Catalog of fgcm zeropoint data from fit cycle {cycle}",
193 name=f
"fgcmZeropoints{cycle}",
194 storageClass=
"Catalog",
195 dimensions=(
"instrument",),
197 vars()[f
"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
198 doc=f
"Catalog of atmospheric fit parameters from fit cycle {cycle}",
199 name=f
"fgcmAtmosphereParameters{cycle}",
200 storageClass=
"Catalog",
201 dimensions=(
"instrument",),
203 vars()[f
"fgcmStandardStars{cycle}"] = connectionTypes.Output(
204 doc=f
"Catalog of standard star magnitudes from fit cycle {cycle}",
205 name=f
"fgcmStandardStars{cycle}",
206 storageClass=
"SimpleCatalog",
207 dimensions=(
"instrument",),
210 def __init__(self, *, config=None):
211 super().__init__(config=config)
213 if not config.doReferenceCalibration:
214 self.inputs.remove(
"fgcmReferenceStars")
216 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
217 raise ValueError(
"cycleNumber must be of integer format")
218 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
219 raise ValueError(
"previousCycleNumber must be of integer format")
220 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
221 raise ValueError(
"previousCycleNumber must be 1 less than cycleNumber")
223 if int(config.connections.cycleNumber) == 0:
224 self.prerequisiteInputs.remove(
"fgcmFlaggedStarsInput")
225 self.prerequisiteInputs.remove(
"fgcmFitParametersInput")
227 if not self.config.doMultipleCycles:
229 if not self.config.isFinalCycle
and not self.config.outputStandardsBeforeFinalCycle:
230 self.outputs.remove(
"fgcmStandardStars")
232 if not self.config.isFinalCycle
and not self.config.outputZeropointsBeforeFinalCycle:
233 self.outputs.remove(
"fgcmZeropoints")
234 self.outputs.remove(
"fgcmAtmosphereParameters")
237 for cycle
in range(0, MULTIPLE_CYCLES_MAX):
238 self.outputs.remove(f
"fgcmFitParameters{cycle}")
239 self.outputs.remove(f
"fgcmFlaggedStars{cycle}")
240 self.outputs.remove(f
"fgcmZeropoints{cycle}")
241 self.outputs.remove(f
"fgcmAtmosphereParameters{cycle}")
242 self.outputs.remove(f
"fgcmStandardStars{cycle}")
247 self.outputs.remove(
"fgcmFitParameters")
248 self.outputs.remove(
"fgcmFlaggedStars")
249 self.outputs.remove(
"fgcmZeropoints")
250 self.outputs.remove(
"fgcmAtmosphereParameters")
251 self.outputs.remove(
"fgcmStandardStars")
254 for cycle
in range(self.config.multipleCyclesFinalCycleNumber + 1,
255 MULTIPLE_CYCLES_MAX):
256 self.outputs.remove(f
"fgcmFitParameters{cycle}")
257 self.outputs.remove(f
"fgcmFlaggedStars{cycle}")
258 self.outputs.remove(f
"fgcmZeropoints{cycle}")
259 self.outputs.remove(f
"fgcmAtmosphereParameters{cycle}")
260 self.outputs.remove(f
"fgcmStandardStars{cycle}")
263 for cycle
in range(self.config.multipleCyclesFinalCycleNumber):
264 if not self.config.outputZeropointsBeforeFinalCycle:
265 self.outputs.remove(f
"fgcmZeropoints{cycle}")
266 self.outputs.remove(f
"fgcmAtmosphereParameters{cycle}")
267 if not self.config.outputStandardsBeforeFinalCycle:
268 self.outputs.remove(f
"fgcmStandardStars{cycle}")
271 class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
272 pipelineConnections=FgcmFitCycleConnections):
273 """Config for FgcmFitCycle"""
275 doMultipleCycles = pexConfig.Field(
276 doc=
"Run multiple fit cycles in one task",
280 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
281 doc=(
"Final cycle number in multiple cycle mode. The initial cycle "
282 "is 0, with limited parameters fit. The next cycle is 1 with "
283 "full parameter fit. The final cycle is a clean-up with no "
284 "parameters fit. There will be a total of "
285 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
286 "cycle number cannot be less than 2."),
290 max=MULTIPLE_CYCLES_MAX,
293 bands = pexConfig.ListField(
294 doc=
"Bands to run calibration",
298 fitBands = pexConfig.ListField(
299 doc=(
"Bands to use in atmospheric fit. The bands not listed here will have "
300 "the atmosphere constrained from the 'fitBands' on the same night. "
301 "Must be a subset of `config.bands`"),
305 requiredBands = pexConfig.ListField(
306 doc=(
"Bands that are required for a star to be considered a calibration star. "
307 "Must be a subset of `config.bands`"),
314 physicalFilterMap = pexConfig.DictField(
315 doc=
"Mapping from 'physicalFilter' to band.",
320 doReferenceCalibration = pexConfig.Field(
321 doc=
"Use reference catalog as additional constraint on calibration",
325 refStarSnMin = pexConfig.Field(
326 doc=
"Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
330 refStarOutlierNSig = pexConfig.Field(
331 doc=(
"Number of sigma compared to average mag for reference star to be considered an outlier. "
332 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
336 applyRefStarColorCuts = pexConfig.Field(
337 doc=
"Apply color cuts to reference stars?",
341 nCore = pexConfig.Field(
342 doc=
"Number of cores to use",
346 nStarPerRun = pexConfig.Field(
347 doc=
"Number of stars to run in each chunk",
351 nExpPerRun = pexConfig.Field(
352 doc=
"Number of exposures to run in each chunk",
356 reserveFraction = pexConfig.Field(
357 doc=
"Fraction of stars to reserve for testing",
361 freezeStdAtmosphere = pexConfig.Field(
362 doc=
"Freeze atmosphere parameters to standard (for testing)",
366 precomputeSuperStarInitialCycle = pexConfig.Field(
367 doc=
"Precompute superstar flat for initial cycle",
371 superStarSubCcdDict = pexConfig.DictField(
372 doc=(
"Per-band specification on whether to compute superstar flat on sub-ccd scale. "
373 "Must have one entry per band."),
378 superStarSubCcdChebyshevOrder = pexConfig.Field(
379 doc=(
"Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
380 "Global default is first-order polynomials, and should be overridden "
381 "on a camera-by-camera basis depending on the ISR."),
385 superStarSubCcdTriangular = pexConfig.Field(
386 doc=(
"Should the sub-ccd superstar chebyshev matrix be triangular to "
387 "suppress high-order cross terms?"),
391 superStarSigmaClip = pexConfig.Field(
392 doc=
"Number of sigma to clip outliers when selecting for superstar flats",
396 focalPlaneSigmaClip = pexConfig.Field(
397 doc=
"Number of sigma to clip outliers per focal-plane.",
401 ccdGraySubCcdDict = pexConfig.DictField(
402 doc=(
"Per-band specification on whether to compute achromatic per-ccd residual "
403 "('ccd gray') on a sub-ccd scale."),
408 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
409 doc=
"Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
413 ccdGraySubCcdTriangular = pexConfig.Field(
414 doc=(
"Should the sub-ccd gray chebyshev matrix be triangular to "
415 "suppress high-order cross terms?"),
419 ccdGrayFocalPlaneDict = pexConfig.DictField(
420 doc=(
"Per-band specification on whether to compute focal-plane residual "
421 "('ccd gray') corrections."),
426 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
427 doc=(
"Minimum number of 'good' CCDs required to perform focal-plane "
428 "gray corrections. If there are fewer good CCDs then the gray "
429 "correction is computed per-ccd."),
433 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
434 doc=
"Order of the 2D chebyshev polynomials for focal plane fit.",
438 cycleNumber = pexConfig.Field(
439 doc=(
"FGCM fit cycle number. This is automatically incremented after each run "
440 "and stage of outlier rejection. See cookbook for details."),
444 isFinalCycle = pexConfig.Field(
445 doc=(
"Is this the final cycle of the fitting? Will automatically compute final "
446 "selection of stars and photometric exposures, and will output zeropoints "
447 "and standard stars for use in fgcmOutputProducts"),
451 maxIterBeforeFinalCycle = pexConfig.Field(
452 doc=(
"Maximum fit iterations, prior to final cycle. The number of iterations "
453 "will always be 0 in the final cycle for cleanup and final selection."),
457 deltaMagBkgOffsetPercentile = pexConfig.Field(
458 doc=(
"Percentile brightest stars on a visit/ccd to use to compute net "
459 "offset from local background subtraction."),
463 deltaMagBkgPerCcd = pexConfig.Field(
464 doc=(
"Compute net offset from local background subtraction per-ccd? "
465 "Otherwise, use computation per visit."),
469 utBoundary = pexConfig.Field(
470 doc=
"Boundary (in UTC) from day-to-day",
474 washMjds = pexConfig.ListField(
475 doc=
"Mirror wash MJDs",
479 epochMjds = pexConfig.ListField(
480 doc=
"Epoch boundaries in MJD",
484 minObsPerBand = pexConfig.Field(
485 doc=
"Minimum good observations per band",
491 latitude = pexConfig.Field(
492 doc=
"Observatory latitude",
496 brightObsGrayMax = pexConfig.Field(
497 doc=
"Maximum gray extinction to be considered bright observation",
501 minStarPerCcd = pexConfig.Field(
502 doc=(
"Minimum number of good stars per CCD to be used in calibration fit. "
503 "CCDs with fewer stars will have their calibration estimated from other "
504 "CCDs in the same visit, with zeropoint error increased accordingly."),
508 minCcdPerExp = pexConfig.Field(
509 doc=(
"Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
510 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
514 maxCcdGrayErr = pexConfig.Field(
515 doc=
"Maximum error on CCD gray offset to be considered photometric",
519 minStarPerExp = pexConfig.Field(
520 doc=(
"Minimum number of good stars per exposure/visit to be used in calibration fit. "
521 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
525 minExpPerNight = pexConfig.Field(
526 doc=
"Minimum number of good exposures/visits to consider a partly photometric night",
530 expGrayInitialCut = pexConfig.Field(
531 doc=(
"Maximum exposure/visit gray value for initial selection of possible photometric "
536 expGrayPhotometricCutDict = pexConfig.DictField(
537 doc=(
"Per-band specification on maximum (negative) achromatic exposure residual "
538 "('gray term') for a visit to be considered photometric. Must have one "
539 "entry per band. Broad-band filters should be -0.05."),
544 expGrayHighCutDict = pexConfig.DictField(
545 doc=(
"Per-band specification on maximum (positive) achromatic exposure residual "
546 "('gray term') for a visit to be considered photometric. Must have one "
547 "entry per band. Broad-band filters should be 0.2."),
552 expGrayRecoverCut = pexConfig.Field(
553 doc=(
"Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
554 "Visits with more gray extinction will only get CCD zeropoints if there are "
555 "sufficient star observations (minStarPerCcd) on that CCD."),
559 expVarGrayPhotometricCutDict = pexConfig.DictField(
560 doc=(
"Per-band specification on maximum exposure variance to be considered possibly "
561 "photometric. Must have one entry per band. Broad-band filters should be "
567 expGrayErrRecoverCut = pexConfig.Field(
568 doc=(
"Maximum exposure gray error to be able to recover bad ccds via interpolation. "
569 "Visits with more gray variance will only get CCD zeropoints if there are "
570 "sufficient star observations (minStarPerCcd) on that CCD."),
574 aperCorrFitNBins = pexConfig.Field(
575 doc=(
"Number of aperture bins used in aperture correction fit. When set to 0"
576 "no fit will be performed, and the config.aperCorrInputSlopes will be "
577 "used if available."),
581 aperCorrInputSlopeDict = pexConfig.DictField(
582 doc=(
"Per-band specification of aperture correction input slope parameters. These "
583 "are used on the first fit iteration, and aperture correction parameters will "
584 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
585 "to set this when there is insufficient data to fit the parameters (e.g. "
591 sedboundaryterms = pexConfig.ConfigField(
592 doc=
"Mapping from bands to SED boundary term names used is sedterms.",
593 dtype=SedboundarytermDict,
595 sedterms = pexConfig.ConfigField(
596 doc=
"Mapping from terms to bands for fgcm linear SED approximations.",
599 sigFgcmMaxErr = pexConfig.Field(
600 doc=
"Maximum mag error for fitting sigma_FGCM",
604 sigFgcmMaxEGrayDict = pexConfig.DictField(
605 doc=(
"Per-band specification for maximum (absolute) achromatic residual (gray value) "
606 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
612 ccdGrayMaxStarErr = pexConfig.Field(
613 doc=(
"Maximum error on a star observation to use in ccd gray (achromatic residual) "
618 approxThroughputDict = pexConfig.DictField(
619 doc=(
"Per-band specification of the approximate overall throughput at the start of "
620 "calibration observations. Must have one entry per band. Typically should "
626 sigmaCalRange = pexConfig.ListField(
627 doc=
"Allowed range for systematic error floor estimation",
629 default=(0.001, 0.003),
631 sigmaCalFitPercentile = pexConfig.ListField(
632 doc=
"Magnitude percentile range to fit systematic error floor",
634 default=(0.05, 0.15),
636 sigmaCalPlotPercentile = pexConfig.ListField(
637 doc=
"Magnitude percentile range to plot systematic error floor",
639 default=(0.05, 0.95),
641 sigma0Phot = pexConfig.Field(
642 doc=
"Systematic error floor for all zeropoints",
646 mapLongitudeRef = pexConfig.Field(
647 doc=
"Reference longitude for plotting maps",
651 mapNSide = pexConfig.Field(
652 doc=
"Healpix nside for plotting maps",
656 outfileBase = pexConfig.Field(
657 doc=
"Filename start for plot output files",
661 starColorCuts = pexConfig.ListField(
662 doc=
"Encoded star-color cuts (to be cleaned up)",
664 default=(
"NO_DATA",),
666 colorSplitBands = pexConfig.ListField(
667 doc=
"Band names to use to split stars by color. Must have 2 entries.",
672 modelMagErrors = pexConfig.Field(
673 doc=
"Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
677 useQuadraticPwv = pexConfig.Field(
678 doc=
"Model PWV with a quadratic term for variation through the night?",
682 instrumentParsPerBand = pexConfig.Field(
683 doc=(
"Model instrumental parameters per band? "
684 "Otherwise, instrumental parameters (QE changes with time) are "
685 "shared among all bands."),
689 instrumentSlopeMinDeltaT = pexConfig.Field(
690 doc=(
"Minimum time change (in days) between observations to use in constraining "
691 "instrument slope."),
695 fitMirrorChromaticity = pexConfig.Field(
696 doc=
"Fit (intraband) mirror chromatic term?",
700 coatingMjds = pexConfig.ListField(
701 doc=
"Mirror coating dates in MJD",
705 outputStandardsBeforeFinalCycle = pexConfig.Field(
706 doc=
"Output standard stars prior to final cycle? Used in debugging.",
710 outputZeropointsBeforeFinalCycle = pexConfig.Field(
711 doc=
"Output standard stars prior to final cycle? Used in debugging.",
715 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
716 doc=(
"Per-band specification on whether to use star repeatability (instead of exposures) "
717 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
722 autoPhotometricCutNSig = pexConfig.Field(
723 doc=(
"Number of sigma for automatic computation of (low) photometric cut. "
724 "Cut is based on exposure gray width (per band), unless "
725 "useRepeatabilityForExpGrayCuts is set, in which case the star "
726 "repeatability is used (also per band)."),
730 autoHighCutNSig = pexConfig.Field(
731 doc=(
"Number of sigma for automatic computation of (high) outlier cut. "
732 "Cut is based on exposure gray width (per band), unless "
733 "useRepeatabilityForExpGrayCuts is set, in which case the star "
734 "repeatability is used (also per band)."),
738 quietMode = pexConfig.Field(
739 doc=
"Be less verbose with logging.",
743 doPlots = pexConfig.Field(
744 doc=
"Make fgcm QA plots.",
748 randomSeed = pexConfig.Field(
749 doc=
"Random seed for fgcm for consistency in tests.",
758 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
759 msg =
"cycleNumber in template must be connections.previousCycleNumber + 1"
760 raise RuntimeError(msg)
761 if self.connections.cycleNumber != str(self.cycleNumber):
762 msg =
"cycleNumber in template must be equal to connections.cycleNumber"
763 raise RuntimeError(msg)
765 for band
in self.fitBands:
766 if band
not in self.bands:
767 msg =
'fitBand %s not in bands' % (band)
768 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
769 for band
in self.requiredBands:
770 if band
not in self.bands:
771 msg =
'requiredBand %s not in bands' % (band)
772 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
773 for band
in self.colorSplitBands:
774 if band
not in self.bands:
775 msg =
'colorSplitBand %s not in bands' % (band)
776 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
777 for band
in self.bands:
778 if band
not in self.superStarSubCcdDict:
779 msg =
'band %s not in superStarSubCcdDict' % (band)
780 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
782 if band
not in self.ccdGraySubCcdDict:
783 msg =
'band %s not in ccdGraySubCcdDict' % (band)
784 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
786 if band
not in self.expGrayPhotometricCutDict:
787 msg =
'band %s not in expGrayPhotometricCutDict' % (band)
788 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
790 if band
not in self.expGrayHighCutDict:
791 msg =
'band %s not in expGrayHighCutDict' % (band)
792 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
794 if band
not in self.expVarGrayPhotometricCutDict:
795 msg =
'band %s not in expVarGrayPhotometricCutDict' % (band)
796 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
798 if band
not in self.sigFgcmMaxEGrayDict:
799 msg =
'band %s not in sigFgcmMaxEGrayDict' % (band)
800 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
802 if band
not in self.approxThroughputDict:
803 msg =
'band %s not in approxThroughputDict' % (band)
804 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
806 if band
not in self.useRepeatabilityForExpGrayCutsDict:
807 msg =
'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
808 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
812 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
813 """Subclass of TaskRunner for fgcmFitCycleTask
815 fgcmFitCycleTask.run() takes one argument, the butler, and uses
816 stars and visits previously extracted from dataRefs by
818 This Runner does not perform any dataRef parallelization, but the FGCM
819 code called by the Task uses python multiprocessing (see the "ncores"
824 def getTargetList(parsedCmd):
826 Return a list with one element, the butler.
828 return [parsedCmd.butler]
830 def __call__(self, butler):
834 butler: `lsst.daf.persistence.Butler`
838 exitStatus: `list` with `pipeBase.Struct`
839 exitStatus (0: success; 1: failure)
842 task = self.TaskClass(config=self.config, log=self.log)
846 task.runDataRef(butler)
849 task.runDataRef(butler)
850 except Exception
as e:
852 task.log.fatal(
"Failed: %s" % e)
853 if not isinstance(e, pipeBase.TaskError):
854 traceback.print_exc(file=sys.stderr)
856 task.writeMetadata(butler)
859 return [pipeBase.Struct(exitStatus=exitStatus)]
861 def run(self, parsedCmd):
863 Run the task, with no multiprocessing
867 parsedCmd: ArgumentParser parsed command line
872 if self.precall(parsedCmd):
873 targetList = self.getTargetList(parsedCmd)
875 resultList = self(targetList[0])
880 class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
882 Run Single fit cycle for FGCM global calibration
885 ConfigClass = FgcmFitCycleConfig
886 RunnerClass = FgcmFitCycleRunner
887 _DefaultName =
"fgcmFitCycle"
889 def __init__(self, butler=None, initInputs=None, **kwargs):
890 super().__init__(**kwargs)
893 def _getMetadataName(self):
896 def runQuantum(self, butlerQC, inputRefs, outputRefs):
897 camera = butlerQC.get(inputRefs.camera)
901 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
902 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
903 dataRefDict[
'fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
904 dataRefDict[
'fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
905 dataRefDict[
'fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
906 if self.config.doReferenceCalibration:
907 dataRefDict[
'fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
908 if self.config.cycleNumber > 0:
909 dataRefDict[
'fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
910 dataRefDict[
'fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
912 fgcmDatasetDict =
None
913 if self.config.doMultipleCycles:
915 config = copy.copy(self.config)
916 config.update(cycleNumber=0)
917 for cycle
in range(self.config.multipleCyclesFinalCycleNumber + 1):
918 if cycle == self.config.multipleCyclesFinalCycleNumber:
919 config.update(isFinalCycle=
True)
922 dataRefDict[
'fgcmFlaggedStars'] = fgcmDatasetDict[
'fgcmFlaggedStars']
923 dataRefDict[
'fgcmFitParameters'] = fgcmDatasetDict[
'fgcmFitParameters']
925 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
926 butlerQC.put(fgcmDatasetDict[
'fgcmFitParameters'],
927 getattr(outputRefs, f
'fgcmFitParameters{cycle}'))
928 butlerQC.put(fgcmDatasetDict[
'fgcmFlaggedStars'],
929 getattr(outputRefs, f
'fgcmFlaggedStars{cycle}'))
930 if self.outputZeropoints:
931 butlerQC.put(fgcmDatasetDict[
'fgcmZeropoints'],
932 getattr(outputRefs, f
'fgcmZeropoints{cycle}'))
933 butlerQC.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'],
934 getattr(outputRefs, f
'fgcmAtmosphereParameters{cycle}'))
935 if self.outputStandards:
936 butlerQC.put(fgcmDatasetDict[
'fgcmStandardStars'],
937 getattr(outputRefs, f
'fgcmStandardStars{cycle}'))
940 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
942 butlerQC.put(fgcmDatasetDict[
'fgcmFitParameters'], outputRefs.fgcmFitParameters)
943 butlerQC.put(fgcmDatasetDict[
'fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
944 if self.outputZeropoints:
945 butlerQC.put(fgcmDatasetDict[
'fgcmZeropoints'], outputRefs.fgcmZeropoints)
946 butlerQC.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
947 if self.outputStandards:
948 butlerQC.put(fgcmDatasetDict[
'fgcmStandardStars'], outputRefs.fgcmStandardStars)
951 def runDataRef(self, butler):
953 Run a single fit cycle for FGCM
957 butler: `lsst.daf.persistence.Butler`
959 self._checkDatasetsExist(butler)
962 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
963 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
964 dataRefDict[
'fgcmStarObservations'] = butler.dataRef(
'fgcmStarObservations')
965 dataRefDict[
'fgcmStarIds'] = butler.dataRef(
'fgcmStarIds')
966 dataRefDict[
'fgcmStarIndices'] = butler.dataRef(
'fgcmStarIndices')
967 if self.config.doReferenceCalibration:
968 dataRefDict[
'fgcmReferenceStars'] = butler.dataRef(
'fgcmReferenceStars')
969 if self.config.cycleNumber > 0:
970 lastCycle = self.config.cycleNumber - 1
971 dataRefDict[
'fgcmFlaggedStars'] = butler.dataRef(
'fgcmFlaggedStars',
973 dataRefDict[
'fgcmFitParameters'] = butler.dataRef(
'fgcmFitParameters',
976 camera = butler.get(
'camera')
977 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
979 butler.put(fgcmDatasetDict[
'fgcmFitParameters'],
'fgcmFitParameters',
980 fgcmcycle=self.config.cycleNumber)
981 butler.put(fgcmDatasetDict[
'fgcmFlaggedStars'],
'fgcmFlaggedStars',
982 fgcmcycle=self.config.cycleNumber)
983 if self.outputZeropoints:
984 butler.put(fgcmDatasetDict[
'fgcmZeropoints'],
'fgcmZeropoints',
985 fgcmcycle=self.config.cycleNumber)
986 butler.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'],
'fgcmAtmosphereParameters',
987 fgcmcycle=self.config.cycleNumber)
988 if self.outputStandards:
989 butler.put(fgcmDatasetDict[
'fgcmStandardStars'],
'fgcmStandardStars',
990 fgcmcycle=self.config.cycleNumber)
992 def writeConfig(self, butler, clobber=False, doBackup=True):
993 """Write the configuration used for processing the data, or check that an existing
994 one is equal to the new one if present. This is an override of the regular
995 version from pipe_base that knows about fgcmcycle.
999 butler : `lsst.daf.persistence.Butler`
1000 Data butler used to write the config. The config is written to dataset type
1001 `CmdLineTask._getConfigName`.
1002 clobber : `bool`, optional
1003 A boolean flag that controls what happens if a config already has been saved:
1004 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1005 - `False`: raise `TaskError` if this config does not match the existing config.
1006 doBackup : `bool`, optional
1007 Set to `True` to backup the config files if clobbering.
1009 configName = self._getConfigName()
1010 if configName
is None:
1013 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1014 elif butler.datasetExists(configName, write=
True, fgcmcycle=self.config.cycleNumber):
1017 oldConfig = butler.get(configName, immediate=
True, fgcmcycle=self.config.cycleNumber)
1018 except Exception
as exc:
1019 raise type(exc)(
"Unable to read stored config file %s (%s); consider using --clobber-config" %
1022 def logConfigMismatch(msg):
1023 self.log.fatal(
"Comparing configuration: %s", msg)
1025 if not self.config.compare(oldConfig, shortcut=
False, output=logConfigMismatch):
1026 raise pipeBase.TaskError(
1027 f
"Config does not match existing task config {configName!r} on disk; tasks configurations"
1028 " must be consistent within the same output repo (override with --clobber-config)")
1030 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1032 def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1038 camera : `lsst.afw.cameraGeom.Camera`
1039 dataRefDict : `dict`
1040 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1041 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1042 dataRef dictionary with keys:
1044 ``"fgcmLookUpTable"``
1045 dataRef for the FGCM look-up table.
1046 ``"fgcmVisitCatalog"``
1047 dataRef for visit summary catalog.
1048 ``"fgcmStarObservations"``
1049 dataRef for star observation catalog.
1051 dataRef for star id catalog.
1052 ``"fgcmStarIndices"``
1053 dataRef for star index catalog.
1054 ``"fgcmReferenceStars"``
1055 dataRef for matched reference star catalog.
1056 ``"fgcmFlaggedStars"``
1057 dataRef for flagged star catalog.
1058 ``"fgcmFitParameters"``
1059 dataRef for fit parameter catalog.
1060 config : `lsst.pex.config.Config`, optional
1061 Configuration to use to override self.config.
1065 fgcmDatasetDict : `dict`
1066 Dictionary of datasets to persist.
1068 if config
is not None:
1071 _config = self.config
1074 self.maxIter = _config.maxIterBeforeFinalCycle
1075 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1076 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1077 self.resetFitParameters =
True
1079 if _config.isFinalCycle:
1084 self.outputStandards =
True
1085 self.outputZeropoints =
True
1086 self.resetFitParameters =
False
1088 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1090 dict(_config.physicalFilterMap))
1094 self.maxIter, self.resetFitParameters,
1095 self.outputZeropoints,
1096 lutIndexVals[0][
'FILTERNAMES'])
1099 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
1107 noFitsDict = {
'lutIndex': lutIndexVals,
1109 'expInfo': fgcmExpInfo,
1110 'ccdOffsets': ccdOffsets}
1113 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
1114 noFitsDict=noFitsDict, noOutput=
True)
1117 if (fgcmFitCycle.initialCycle):
1119 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1123 if isinstance(dataRefDict[
'fgcmFitParameters'], afwTable.BaseCatalog):
1124 parCat = dataRefDict[
'fgcmFitParameters']
1126 parCat = dataRefDict[
'fgcmFitParameters'].get()
1127 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1129 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1136 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1138 starObs = dataRefDict[
'fgcmStarObservations'].get()
1139 starIds = dataRefDict[
'fgcmStarIds'].get()
1140 starIndices = dataRefDict[
'fgcmStarIndices'].get()
1143 if 'fgcmFlaggedStars' in dataRefDict:
1144 if isinstance(dataRefDict[
'fgcmFlaggedStars'], afwTable.BaseCatalog):
1145 flaggedStars = dataRefDict[
'fgcmFlaggedStars']
1147 flaggedStars = dataRefDict[
'fgcmFlaggedStars'].get()
1148 flagId = flaggedStars[
'objId'][:]
1149 flagFlag = flaggedStars[
'objFlag'][:]
1155 if _config.doReferenceCalibration:
1156 refStars = dataRefDict[
'fgcmReferenceStars'].get()
1160 _config.physicalFilterMap)
1161 refId = refStars[
'fgcm_id'][:]
1171 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'], starObs[
'visit'][starIndices[
'obsIndex']])
1183 conv = starObs[0][
'ra'].asDegrees() / float(starObs[0][
'ra'])
1185 fgcmStars.loadStars(fgcmPars,
1186 starObs[
'visit'][starIndices[
'obsIndex']],
1187 starObs[
'ccd'][starIndices[
'obsIndex']],
1188 starObs[
'ra'][starIndices[
'obsIndex']] * conv,
1189 starObs[
'dec'][starIndices[
'obsIndex']] * conv,
1190 starObs[
'instMag'][starIndices[
'obsIndex']],
1191 starObs[
'instMagErr'][starIndices[
'obsIndex']],
1192 fgcmExpInfo[
'FILTERNAME'][visitIndex],
1193 starIds[
'fgcm_id'][:],
1196 starIds[
'obsArrIndex'][:],
1198 obsX=starObs[
'x'][starIndices[
'obsIndex']],
1199 obsY=starObs[
'y'][starIndices[
'obsIndex']],
1200 obsDeltaMagBkg=starObs[
'deltaMagBkg'][starIndices[
'obsIndex']],
1201 psfCandidate=starObs[
'psf_candidate'][starIndices[
'obsIndex']],
1204 refMagErr=refMagErr,
1222 fgcmFitCycle.setLUT(fgcmLut)
1223 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1224 fgcmFitCycle.setPars(fgcmPars)
1227 fgcmFitCycle.finishSetup()
1236 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1241 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i])
for
1242 i, b
in enumerate(_config.bands)}
1243 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i])
for
1244 i, band
in enumerate(_config.bands)}
1246 outConfig = copy.copy(_config)
1247 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1248 precomputeSuperStarInitialCycle=
False,
1249 freezeStdAtmosphere=
False,
1250 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1251 expGrayHighCutDict=updatedHighCutDict)
1253 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1254 cycleNumber=str(_config.cycleNumber + 1))
1256 configFileName =
'%s_cycle%02d_config.py' % (outConfig.outfileBase,
1257 outConfig.cycleNumber)
1258 outConfig.save(configFileName)
1260 if _config.isFinalCycle == 1:
1262 self.log.info(
"Everything is in place to run fgcmOutputProducts.py")
1264 self.log.info(
"Saved config for next cycle to %s" % (configFileName))
1265 self.log.info(
"Be sure to look at:")
1266 self.log.info(
" config.expGrayPhotometricCut")
1267 self.log.info(
" config.expGrayHighCut")
1268 self.log.info(
"If you are satisfied with the fit, please set:")
1269 self.log.info(
" config.isFinalCycle = True")
1271 fgcmFitCycle.freeSharedMemory()
1273 return fgcmDatasetDict, outConfig
1275 def _checkDatasetsExist(self, butler):
1277 Check if necessary datasets exist to run fgcmFitCycle
1281 butler: `lsst.daf.persistence.Butler`
1286 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1287 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1288 If cycleNumber > 0, then also checks for fgcmFitParameters,
1292 if not butler.datasetExists(
'fgcmVisitCatalog'):
1293 raise RuntimeError(
"Could not find fgcmVisitCatalog in repo!")
1294 if not butler.datasetExists(
'fgcmStarObservations'):
1295 raise RuntimeError(
"Could not find fgcmStarObservations in repo!")
1296 if not butler.datasetExists(
'fgcmStarIds'):
1297 raise RuntimeError(
"Could not find fgcmStarIds in repo!")
1298 if not butler.datasetExists(
'fgcmStarIndices'):
1299 raise RuntimeError(
"Could not find fgcmStarIndices in repo!")
1300 if not butler.datasetExists(
'fgcmLookUpTable'):
1301 raise RuntimeError(
"Could not find fgcmLookUpTable in repo!")
1304 if (self.config.cycleNumber > 0):
1305 if not butler.datasetExists(
'fgcmFitParameters',
1306 fgcmcycle=self.config.cycleNumber-1):
1307 raise RuntimeError(
"Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1308 (self.config.cycleNumber-1))
1309 if not butler.datasetExists(
'fgcmFlaggedStars',
1310 fgcmcycle=self.config.cycleNumber-1):
1311 raise RuntimeError(
"Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1312 (self.config.cycleNumber-1))
1315 if self.config.doReferenceCalibration:
1316 if not butler.datasetExists(
'fgcmReferenceStars'):
1317 raise RuntimeError(
"Could not find fgcmReferenceStars in repo, and "
1318 "doReferenceCalibration is True.")
1320 def _loadParameters(self, parCat):
1322 Load FGCM parameters from a previous fit cycle
1326 parCat : `lsst.afw.table.BaseCatalog`
1327 Parameter catalog in afw table form.
1331 inParInfo: `numpy.ndarray`
1332 Numpy array parameter information formatted for input to fgcm
1333 inParameters: `numpy.ndarray`
1334 Numpy array parameter values formatted for input to fgcm
1335 inSuperStar: `numpy.array`
1336 Superstar flat formatted for input to fgcm
1338 parLutFilterNames = np.array(parCat[0][
'lutFilterNames'].split(
','))
1339 parFitBands = np.array(parCat[0][
'fitBands'].split(
','))
1341 inParInfo = np.zeros(1, dtype=[(
'NCCD',
'i4'),
1342 (
'LUTFILTERNAMES', parLutFilterNames.dtype.str,
1343 (parLutFilterNames.size, )),
1344 (
'FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1345 (
'LNTAUUNIT',
'f8'),
1346 (
'LNTAUSLOPEUNIT',
'f8'),
1347 (
'ALPHAUNIT',
'f8'),
1348 (
'LNPWVUNIT',
'f8'),
1349 (
'LNPWVSLOPEUNIT',
'f8'),
1350 (
'LNPWVQUADRATICUNIT',
'f8'),
1351 (
'LNPWVGLOBALUNIT',
'f8'),
1353 (
'QESYSUNIT',
'f8'),
1354 (
'FILTEROFFSETUNIT',
'f8'),
1355 (
'HASEXTERNALPWV',
'i2'),
1356 (
'HASEXTERNALTAU',
'i2')])
1357 inParInfo[
'NCCD'] = parCat[
'nCcd']
1358 inParInfo[
'LUTFILTERNAMES'][:] = parLutFilterNames
1359 inParInfo[
'FITBANDS'][:] = parFitBands
1360 inParInfo[
'HASEXTERNALPWV'] = parCat[
'hasExternalPwv']
1361 inParInfo[
'HASEXTERNALTAU'] = parCat[
'hasExternalTau']
1363 inParams = np.zeros(1, dtype=[(
'PARALPHA',
'f8', (parCat[
'parAlpha'].size, )),
1364 (
'PARO3',
'f8', (parCat[
'parO3'].size, )),
1365 (
'PARLNTAUINTERCEPT',
'f8',
1366 (parCat[
'parLnTauIntercept'].size, )),
1367 (
'PARLNTAUSLOPE',
'f8',
1368 (parCat[
'parLnTauSlope'].size, )),
1369 (
'PARLNPWVINTERCEPT',
'f8',
1370 (parCat[
'parLnPwvIntercept'].size, )),
1371 (
'PARLNPWVSLOPE',
'f8',
1372 (parCat[
'parLnPwvSlope'].size, )),
1373 (
'PARLNPWVQUADRATIC',
'f8',
1374 (parCat[
'parLnPwvQuadratic'].size, )),
1375 (
'PARQESYSINTERCEPT',
'f8',
1376 (parCat[
'parQeSysIntercept'].size, )),
1377 (
'COMPQESYSSLOPE',
'f8',
1378 (parCat[
'compQeSysSlope'].size, )),
1379 (
'PARFILTEROFFSET',
'f8',
1380 (parCat[
'parFilterOffset'].size, )),
1381 (
'PARFILTEROFFSETFITFLAG',
'i2',
1382 (parCat[
'parFilterOffsetFitFlag'].size, )),
1383 (
'PARRETRIEVEDLNPWVSCALE',
'f8'),
1384 (
'PARRETRIEVEDLNPWVOFFSET',
'f8'),
1385 (
'PARRETRIEVEDLNPWVNIGHTLYOFFSET',
'f8',
1386 (parCat[
'parRetrievedLnPwvNightlyOffset'].size, )),
1387 (
'COMPABSTHROUGHPUT',
'f8',
1388 (parCat[
'compAbsThroughput'].size, )),
1389 (
'COMPREFOFFSET',
'f8',
1390 (parCat[
'compRefOffset'].size, )),
1391 (
'COMPREFSIGMA',
'f8',
1392 (parCat[
'compRefSigma'].size, )),
1393 (
'COMPMIRRORCHROMATICITY',
'f8',
1394 (parCat[
'compMirrorChromaticity'].size, )),
1395 (
'MIRRORCHROMATICITYPIVOT',
'f8',
1396 (parCat[
'mirrorChromaticityPivot'].size, )),
1397 (
'COMPMEDIANSEDSLOPE',
'f8',
1398 (parCat[
'compMedianSedSlope'].size, )),
1399 (
'COMPAPERCORRPIVOT',
'f8',
1400 (parCat[
'compAperCorrPivot'].size, )),
1401 (
'COMPAPERCORRSLOPE',
'f8',
1402 (parCat[
'compAperCorrSlope'].size, )),
1403 (
'COMPAPERCORRSLOPEERR',
'f8',
1404 (parCat[
'compAperCorrSlopeErr'].size, )),
1405 (
'COMPAPERCORRRANGE',
'f8',
1406 (parCat[
'compAperCorrRange'].size, )),
1407 (
'COMPMODELERREXPTIMEPIVOT',
'f8',
1408 (parCat[
'compModelErrExptimePivot'].size, )),
1409 (
'COMPMODELERRFWHMPIVOT',
'f8',
1410 (parCat[
'compModelErrFwhmPivot'].size, )),
1411 (
'COMPMODELERRSKYPIVOT',
'f8',
1412 (parCat[
'compModelErrSkyPivot'].size, )),
1413 (
'COMPMODELERRPARS',
'f8',
1414 (parCat[
'compModelErrPars'].size, )),
1415 (
'COMPEXPGRAY',
'f8',
1416 (parCat[
'compExpGray'].size, )),
1417 (
'COMPVARGRAY',
'f8',
1418 (parCat[
'compVarGray'].size, )),
1419 (
'COMPEXPDELTAMAGBKG',
'f8',
1420 (parCat[
'compExpDeltaMagBkg'].size, )),
1421 (
'COMPNGOODSTARPEREXP',
'i4',
1422 (parCat[
'compNGoodStarPerExp'].size, )),
1423 (
'COMPSIGFGCM',
'f8',
1424 (parCat[
'compSigFgcm'].size, )),
1425 (
'COMPSIGMACAL',
'f8',
1426 (parCat[
'compSigmaCal'].size, )),
1427 (
'COMPRETRIEVEDLNPWV',
'f8',
1428 (parCat[
'compRetrievedLnPwv'].size, )),
1429 (
'COMPRETRIEVEDLNPWVRAW',
'f8',
1430 (parCat[
'compRetrievedLnPwvRaw'].size, )),
1431 (
'COMPRETRIEVEDLNPWVFLAG',
'i2',
1432 (parCat[
'compRetrievedLnPwvFlag'].size, )),
1433 (
'COMPRETRIEVEDTAUNIGHT',
'f8',
1434 (parCat[
'compRetrievedTauNight'].size, ))])
1436 inParams[
'PARALPHA'][:] = parCat[
'parAlpha'][0, :]
1437 inParams[
'PARO3'][:] = parCat[
'parO3'][0, :]
1438 inParams[
'PARLNTAUINTERCEPT'][:] = parCat[
'parLnTauIntercept'][0, :]
1439 inParams[
'PARLNTAUSLOPE'][:] = parCat[
'parLnTauSlope'][0, :]
1440 inParams[
'PARLNPWVINTERCEPT'][:] = parCat[
'parLnPwvIntercept'][0, :]
1441 inParams[
'PARLNPWVSLOPE'][:] = parCat[
'parLnPwvSlope'][0, :]
1442 inParams[
'PARLNPWVQUADRATIC'][:] = parCat[
'parLnPwvQuadratic'][0, :]
1443 inParams[
'PARQESYSINTERCEPT'][:] = parCat[
'parQeSysIntercept'][0, :]
1444 inParams[
'COMPQESYSSLOPE'][:] = parCat[
'compQeSysSlope'][0, :]
1445 inParams[
'PARFILTEROFFSET'][:] = parCat[
'parFilterOffset'][0, :]
1446 inParams[
'PARFILTEROFFSETFITFLAG'][:] = parCat[
'parFilterOffsetFitFlag'][0, :]
1447 inParams[
'PARRETRIEVEDLNPWVSCALE'] = parCat[
'parRetrievedLnPwvScale']
1448 inParams[
'PARRETRIEVEDLNPWVOFFSET'] = parCat[
'parRetrievedLnPwvOffset']
1449 inParams[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat[
'parRetrievedLnPwvNightlyOffset'][0, :]
1450 inParams[
'COMPABSTHROUGHPUT'][:] = parCat[
'compAbsThroughput'][0, :]
1451 inParams[
'COMPREFOFFSET'][:] = parCat[
'compRefOffset'][0, :]
1452 inParams[
'COMPREFSIGMA'][:] = parCat[
'compRefSigma'][0, :]
1453 inParams[
'COMPMIRRORCHROMATICITY'][:] = parCat[
'compMirrorChromaticity'][0, :]
1454 inParams[
'MIRRORCHROMATICITYPIVOT'][:] = parCat[
'mirrorChromaticityPivot'][0, :]
1455 inParams[
'COMPMEDIANSEDSLOPE'][:] = parCat[
'compMedianSedSlope'][0, :]
1456 inParams[
'COMPAPERCORRPIVOT'][:] = parCat[
'compAperCorrPivot'][0, :]
1457 inParams[
'COMPAPERCORRSLOPE'][:] = parCat[
'compAperCorrSlope'][0, :]
1458 inParams[
'COMPAPERCORRSLOPEERR'][:] = parCat[
'compAperCorrSlopeErr'][0, :]
1459 inParams[
'COMPAPERCORRRANGE'][:] = parCat[
'compAperCorrRange'][0, :]
1460 inParams[
'COMPMODELERREXPTIMEPIVOT'][:] = parCat[
'compModelErrExptimePivot'][0, :]
1461 inParams[
'COMPMODELERRFWHMPIVOT'][:] = parCat[
'compModelErrFwhmPivot'][0, :]
1462 inParams[
'COMPMODELERRSKYPIVOT'][:] = parCat[
'compModelErrSkyPivot'][0, :]
1463 inParams[
'COMPMODELERRPARS'][:] = parCat[
'compModelErrPars'][0, :]
1464 inParams[
'COMPEXPGRAY'][:] = parCat[
'compExpGray'][0, :]
1465 inParams[
'COMPVARGRAY'][:] = parCat[
'compVarGray'][0, :]
1466 inParams[
'COMPEXPDELTAMAGBKG'][:] = parCat[
'compExpDeltaMagBkg'][0, :]
1467 inParams[
'COMPNGOODSTARPEREXP'][:] = parCat[
'compNGoodStarPerExp'][0, :]
1468 inParams[
'COMPSIGFGCM'][:] = parCat[
'compSigFgcm'][0, :]
1469 inParams[
'COMPSIGMACAL'][:] = parCat[
'compSigmaCal'][0, :]
1470 inParams[
'COMPRETRIEVEDLNPWV'][:] = parCat[
'compRetrievedLnPwv'][0, :]
1471 inParams[
'COMPRETRIEVEDLNPWVRAW'][:] = parCat[
'compRetrievedLnPwvRaw'][0, :]
1472 inParams[
'COMPRETRIEVEDLNPWVFLAG'][:] = parCat[
'compRetrievedLnPwvFlag'][0, :]
1473 inParams[
'COMPRETRIEVEDTAUNIGHT'][:] = parCat[
'compRetrievedTauNight'][0, :]
1475 inSuperStar = np.zeros(parCat[
'superstarSize'][0, :], dtype=
'f8')
1476 inSuperStar[:, :, :, :] = parCat[
'superstar'][0, :].reshape(inSuperStar.shape)
1478 return (inParInfo, inParams, inSuperStar)
1480 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1482 Persist FGCM datasets through the butler.
1486 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1487 Fgcm Fit cycle object
1489 fgcmDatasetDict = {}
1492 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1494 parSchema = afwTable.Schema()
1497 lutFilterNameString = comma.join([n.decode(
'utf-8')
1498 for n
in parInfo[
'LUTFILTERNAMES'][0]])
1499 fitBandString = comma.join([n.decode(
'utf-8')
1500 for n
in parInfo[
'FITBANDS'][0]])
1502 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1503 lutFilterNameString, fitBandString)
1504 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1505 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1506 lutFilterNameString, fitBandString)
1508 fgcmDatasetDict[
'fgcmFitParameters'] = parCat
1513 flagStarSchema = self._makeFlagStarSchema()
1514 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1515 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1517 fgcmDatasetDict[
'fgcmFlaggedStars'] = flagStarCat
1520 if self.outputZeropoints:
1521 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
1522 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
1525 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1527 fgcmDatasetDict[
'fgcmZeropoints'] = zptCat
1532 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1534 fgcmDatasetDict[
'fgcmAtmosphereParameters'] = atmCat
1537 if self.outputStandards:
1538 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1540 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
1542 fgcmDatasetDict[
'fgcmStandardStars'] = stdCat
1544 return fgcmDatasetDict
1546 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1547 lutFilterNameString, fitBandString):
1549 Make the parameter persistence schema
1553 parInfo: `numpy.ndarray`
1554 Parameter information returned by fgcm
1555 pars: `numpy.ndarray`
1556 Parameter values returned by fgcm
1557 parSuperStarFlat: `numpy.array`
1558 Superstar flat values returned by fgcm
1559 lutFilterNameString: `str`
1560 Combined string of all the lutFilterNames
1561 fitBandString: `str`
1562 Combined string of all the fitBands
1566 parSchema: `afwTable.schema`
1569 parSchema = afwTable.Schema()
1572 parSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
1573 parSchema.addField(
'lutFilterNames', type=str, doc=
'LUT Filter names in parameter file',
1574 size=len(lutFilterNameString))
1575 parSchema.addField(
'fitBands', type=str, doc=
'Bands that were fit',
1576 size=len(fitBandString))
1577 parSchema.addField(
'lnTauUnit', type=np.float64, doc=
'Step units for ln(AOD)')
1578 parSchema.addField(
'lnTauSlopeUnit', type=np.float64,
1579 doc=
'Step units for ln(AOD) slope')
1580 parSchema.addField(
'alphaUnit', type=np.float64, doc=
'Step units for alpha')
1581 parSchema.addField(
'lnPwvUnit', type=np.float64, doc=
'Step units for ln(pwv)')
1582 parSchema.addField(
'lnPwvSlopeUnit', type=np.float64,
1583 doc=
'Step units for ln(pwv) slope')
1584 parSchema.addField(
'lnPwvQuadraticUnit', type=np.float64,
1585 doc=
'Step units for ln(pwv) quadratic term')
1586 parSchema.addField(
'lnPwvGlobalUnit', type=np.float64,
1587 doc=
'Step units for global ln(pwv) parameters')
1588 parSchema.addField(
'o3Unit', type=np.float64, doc=
'Step units for O3')
1589 parSchema.addField(
'qeSysUnit', type=np.float64, doc=
'Step units for mirror gray')
1590 parSchema.addField(
'filterOffsetUnit', type=np.float64, doc=
'Step units for filter offset')
1591 parSchema.addField(
'hasExternalPwv', type=np.int32, doc=
'Parameters fit using external pwv')
1592 parSchema.addField(
'hasExternalTau', type=np.int32, doc=
'Parameters fit using external tau')
1595 parSchema.addField(
'parAlpha', type=
'ArrayD', doc=
'Alpha parameter vector',
1596 size=pars[
'PARALPHA'].size)
1597 parSchema.addField(
'parO3', type=
'ArrayD', doc=
'O3 parameter vector',
1598 size=pars[
'PARO3'].size)
1599 parSchema.addField(
'parLnTauIntercept', type=
'ArrayD',
1600 doc=
'ln(Tau) intercept parameter vector',
1601 size=pars[
'PARLNTAUINTERCEPT'].size)
1602 parSchema.addField(
'parLnTauSlope', type=
'ArrayD',
1603 doc=
'ln(Tau) slope parameter vector',
1604 size=pars[
'PARLNTAUSLOPE'].size)
1605 parSchema.addField(
'parLnPwvIntercept', type=
'ArrayD', doc=
'ln(pwv) intercept parameter vector',
1606 size=pars[
'PARLNPWVINTERCEPT'].size)
1607 parSchema.addField(
'parLnPwvSlope', type=
'ArrayD', doc=
'ln(pwv) slope parameter vector',
1608 size=pars[
'PARLNPWVSLOPE'].size)
1609 parSchema.addField(
'parLnPwvQuadratic', type=
'ArrayD', doc=
'ln(pwv) quadratic parameter vector',
1610 size=pars[
'PARLNPWVQUADRATIC'].size)
1611 parSchema.addField(
'parQeSysIntercept', type=
'ArrayD', doc=
'Mirror gray intercept parameter vector',
1612 size=pars[
'PARQESYSINTERCEPT'].size)
1613 parSchema.addField(
'compQeSysSlope', type=
'ArrayD', doc=
'Mirror gray slope parameter vector',
1614 size=pars[0][
'COMPQESYSSLOPE'].size)
1615 parSchema.addField(
'parFilterOffset', type=
'ArrayD', doc=
'Filter offset parameter vector',
1616 size=pars[
'PARFILTEROFFSET'].size)
1617 parSchema.addField(
'parFilterOffsetFitFlag', type=
'ArrayI', doc=
'Filter offset parameter fit flag',
1618 size=pars[
'PARFILTEROFFSETFITFLAG'].size)
1619 parSchema.addField(
'parRetrievedLnPwvScale', type=np.float64,
1620 doc=
'Global scale for retrieved ln(pwv)')
1621 parSchema.addField(
'parRetrievedLnPwvOffset', type=np.float64,
1622 doc=
'Global offset for retrieved ln(pwv)')
1623 parSchema.addField(
'parRetrievedLnPwvNightlyOffset', type=
'ArrayD',
1624 doc=
'Nightly offset for retrieved ln(pwv)',
1625 size=pars[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1626 parSchema.addField(
'compAbsThroughput', type=
'ArrayD',
1627 doc=
'Absolute throughput (relative to transmission curves)',
1628 size=pars[
'COMPABSTHROUGHPUT'].size)
1629 parSchema.addField(
'compRefOffset', type=
'ArrayD',
1630 doc=
'Offset between reference stars and calibrated stars',
1631 size=pars[
'COMPREFOFFSET'].size)
1632 parSchema.addField(
'compRefSigma', type=
'ArrayD',
1633 doc=
'Width of reference star/calibrated star distribution',
1634 size=pars[
'COMPREFSIGMA'].size)
1635 parSchema.addField(
'compMirrorChromaticity', type=
'ArrayD',
1636 doc=
'Computed mirror chromaticity terms',
1637 size=pars[
'COMPMIRRORCHROMATICITY'].size)
1638 parSchema.addField(
'mirrorChromaticityPivot', type=
'ArrayD',
1639 doc=
'Mirror chromaticity pivot mjd',
1640 size=pars[
'MIRRORCHROMATICITYPIVOT'].size)
1641 parSchema.addField(
'compMedianSedSlope', type=
'ArrayD',
1642 doc=
'Computed median SED slope (per band)',
1643 size=pars[
'COMPMEDIANSEDSLOPE'].size)
1644 parSchema.addField(
'compAperCorrPivot', type=
'ArrayD', doc=
'Aperture correction pivot',
1645 size=pars[
'COMPAPERCORRPIVOT'].size)
1646 parSchema.addField(
'compAperCorrSlope', type=
'ArrayD', doc=
'Aperture correction slope',
1647 size=pars[
'COMPAPERCORRSLOPE'].size)
1648 parSchema.addField(
'compAperCorrSlopeErr', type=
'ArrayD', doc=
'Aperture correction slope error',
1649 size=pars[
'COMPAPERCORRSLOPEERR'].size)
1650 parSchema.addField(
'compAperCorrRange', type=
'ArrayD', doc=
'Aperture correction range',
1651 size=pars[
'COMPAPERCORRRANGE'].size)
1652 parSchema.addField(
'compModelErrExptimePivot', type=
'ArrayD', doc=
'Model error exptime pivot',
1653 size=pars[
'COMPMODELERREXPTIMEPIVOT'].size)
1654 parSchema.addField(
'compModelErrFwhmPivot', type=
'ArrayD', doc=
'Model error fwhm pivot',
1655 size=pars[
'COMPMODELERRFWHMPIVOT'].size)
1656 parSchema.addField(
'compModelErrSkyPivot', type=
'ArrayD', doc=
'Model error sky pivot',
1657 size=pars[
'COMPMODELERRSKYPIVOT'].size)
1658 parSchema.addField(
'compModelErrPars', type=
'ArrayD', doc=
'Model error parameters',
1659 size=pars[
'COMPMODELERRPARS'].size)
1660 parSchema.addField(
'compExpGray', type=
'ArrayD', doc=
'Computed exposure gray',
1661 size=pars[
'COMPEXPGRAY'].size)
1662 parSchema.addField(
'compVarGray', type=
'ArrayD', doc=
'Computed exposure variance',
1663 size=pars[
'COMPVARGRAY'].size)
1664 parSchema.addField(
'compExpDeltaMagBkg', type=
'ArrayD',
1665 doc=
'Computed exposure offset due to background',
1666 size=pars[
'COMPEXPDELTAMAGBKG'].size)
1667 parSchema.addField(
'compNGoodStarPerExp', type=
'ArrayI',
1668 doc=
'Computed number of good stars per exposure',
1669 size=pars[
'COMPNGOODSTARPEREXP'].size)
1670 parSchema.addField(
'compSigFgcm', type=
'ArrayD', doc=
'Computed sigma_fgcm (intrinsic repeatability)',
1671 size=pars[
'COMPSIGFGCM'].size)
1672 parSchema.addField(
'compSigmaCal', type=
'ArrayD', doc=
'Computed sigma_cal (systematic error floor)',
1673 size=pars[
'COMPSIGMACAL'].size)
1674 parSchema.addField(
'compRetrievedLnPwv', type=
'ArrayD', doc=
'Retrieved ln(pwv) (smoothed)',
1675 size=pars[
'COMPRETRIEVEDLNPWV'].size)
1676 parSchema.addField(
'compRetrievedLnPwvRaw', type=
'ArrayD', doc=
'Retrieved ln(pwv) (raw)',
1677 size=pars[
'COMPRETRIEVEDLNPWVRAW'].size)
1678 parSchema.addField(
'compRetrievedLnPwvFlag', type=
'ArrayI', doc=
'Retrieved ln(pwv) Flag',
1679 size=pars[
'COMPRETRIEVEDLNPWVFLAG'].size)
1680 parSchema.addField(
'compRetrievedTauNight', type=
'ArrayD', doc=
'Retrieved tau (per night)',
1681 size=pars[
'COMPRETRIEVEDTAUNIGHT'].size)
1683 parSchema.addField(
'superstarSize', type=
'ArrayI', doc=
'Superstar matrix size',
1685 parSchema.addField(
'superstar', type=
'ArrayD', doc=
'Superstar matrix (flattened)',
1686 size=parSuperStarFlat.size)
1690 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1691 lutFilterNameString, fitBandString):
1693 Make the FGCM parameter catalog for persistence
1697 parSchema: `lsst.afw.table.Schema`
1698 Parameter catalog schema
1699 pars: `numpy.ndarray`
1700 FGCM parameters to put into parCat
1701 parSuperStarFlat: `numpy.array`
1702 FGCM superstar flat array to put into parCat
1703 lutFilterNameString: `str`
1704 Combined string of all the lutFilterNames
1705 fitBandString: `str`
1706 Combined string of all the fitBands
1710 parCat: `afwTable.BasicCatalog`
1711 Atmosphere and instrumental model parameter catalog for persistence
1714 parCat = afwTable.BaseCatalog(parSchema)
1719 rec = parCat.addNew()
1722 rec[
'nCcd'] = parInfo[
'NCCD']
1723 rec[
'lutFilterNames'] = lutFilterNameString
1724 rec[
'fitBands'] = fitBandString
1726 rec[
'hasExternalPwv'] = 0
1727 rec[
'hasExternalTau'] = 0
1731 scalarNames = [
'parRetrievedLnPwvScale',
'parRetrievedLnPwvOffset']
1733 arrNames = [
'parAlpha',
'parO3',
'parLnTauIntercept',
'parLnTauSlope',
1734 'parLnPwvIntercept',
'parLnPwvSlope',
'parLnPwvQuadratic',
1735 'parQeSysIntercept',
'compQeSysSlope',
1736 'parRetrievedLnPwvNightlyOffset',
'compAperCorrPivot',
1737 'parFilterOffset',
'parFilterOffsetFitFlag',
1738 'compAbsThroughput',
'compRefOffset',
'compRefSigma',
1739 'compMirrorChromaticity',
'mirrorChromaticityPivot',
1740 'compAperCorrSlope',
'compAperCorrSlopeErr',
'compAperCorrRange',
1741 'compModelErrExptimePivot',
'compModelErrFwhmPivot',
1742 'compModelErrSkyPivot',
'compModelErrPars',
1743 'compExpGray',
'compVarGray',
'compNGoodStarPerExp',
'compSigFgcm',
1744 'compSigmaCal',
'compExpDeltaMagBkg',
'compMedianSedSlope',
1745 'compRetrievedLnPwv',
'compRetrievedLnPwvRaw',
'compRetrievedLnPwvFlag',
1746 'compRetrievedTauNight']
1748 for scalarName
in scalarNames:
1749 rec[scalarName] = pars[scalarName.upper()]
1751 for arrName
in arrNames:
1752 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1755 rec[
'superstarSize'][:] = parSuperStarFlat.shape
1756 rec[
'superstar'][:] = parSuperStarFlat.flatten()
1760 def _makeFlagStarSchema(self):
1762 Make the flagged-stars schema
1766 flagStarSchema: `lsst.afw.table.Schema`
1769 flagStarSchema = afwTable.Schema()
1771 flagStarSchema.addField(
'objId', type=np.int32, doc=
'FGCM object id')
1772 flagStarSchema.addField(
'objFlag', type=np.int32, doc=
'FGCM object flag')
1774 return flagStarSchema
1776 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1778 Make the flagged star catalog for persistence
1782 flagStarSchema: `lsst.afw.table.Schema`
1784 flagStarStruct: `numpy.ndarray`
1785 Flagged star structure from fgcm
1789 flagStarCat: `lsst.afw.table.BaseCatalog`
1790 Flagged star catalog for persistence
1793 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1794 flagStarCat.resize(flagStarStruct.size)
1796 flagStarCat[
'objId'][:] = flagStarStruct[
'OBJID']
1797 flagStarCat[
'objFlag'][:] = flagStarStruct[
'OBJFLAG']
def extractReferenceMags(refStars, bands, filterMap)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)