23 """Perform a single fit cycle of FGCM.
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
42 import lsst.pex.config
as pexConfig
43 import lsst.pipe.base
as pipeBase
44 from lsst.pipe.base
import connectionTypes
45 import lsst.afw.table
as afwTable
47 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
48 from .utilities
import extractReferenceMags
49 from .utilities
import computeCcdOffsets, makeZptSchema, makeZptCat
50 from .utilities
import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51 from .sedterms
import SedboundarytermDict, SedtermDict
52 from .utilities
import lookupStaticCalibrations
56 __all__ = [
'FgcmFitCycleConfig',
'FgcmFitCycleTask',
'FgcmFitCycleRunner']
60 dimensions=(
"instrument",),
61 defaultTemplates={
"previousCycleNumber":
"-1",
63 camera = connectionTypes.PrerequisiteInput(
64 doc=
"Camera instrument",
66 storageClass=
"Camera",
67 dimensions=(
"instrument",),
68 lookupFunction=lookupStaticCalibrations,
72 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
73 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
74 "chromatic corrections."),
75 name=
"fgcmLookUpTable",
76 storageClass=
"Catalog",
77 dimensions=(
"instrument",),
81 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
82 doc=
"Catalog of visit information for fgcm",
83 name=
"fgcmVisitCatalog",
84 storageClass=
"Catalog",
85 dimensions=(
"instrument",),
89 fgcmStarObservations = connectionTypes.PrerequisiteInput(
90 doc=
"Catalog of star observations for fgcm",
91 name=
"fgcmStarObservations",
92 storageClass=
"Catalog",
93 dimensions=(
"instrument",),
97 fgcmStarIds = connectionTypes.PrerequisiteInput(
98 doc=
"Catalog of fgcm calibration star IDs",
100 storageClass=
"Catalog",
101 dimensions=(
"instrument",),
105 fgcmStarIndices = connectionTypes.PrerequisiteInput(
106 doc=
"Catalog of fgcm calibration star indices",
107 name=
"fgcmStarIndices",
108 storageClass=
"Catalog",
109 dimensions=(
"instrument",),
113 fgcmReferenceStars = connectionTypes.PrerequisiteInput(
114 doc=
"Catalog of fgcm-matched reference stars",
115 name=
"fgcmReferenceStars",
116 storageClass=
"Catalog",
117 dimensions=(
"instrument",),
121 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
122 doc=
"Catalog of flagged stars for fgcm calibration from previous fit cycle",
123 name=
"fgcmFlaggedStars{previousCycleNumber}",
124 storageClass=
"Catalog",
125 dimensions=(
"instrument",),
129 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
130 doc=
"Catalog of fgcm fit parameters from previous fit cycle",
131 name=
"fgcmFitParameters{previousCycleNumber}",
132 storageClass=
"Catalog",
133 dimensions=(
"instrument",),
137 fgcmFitParameters = connectionTypes.Output(
138 doc=
"Catalog of fgcm fit parameters from current fit cycle",
139 name=
"fgcmFitParameters{cycleNumber}",
140 storageClass=
"Catalog",
141 dimensions=(
"instrument",),
144 fgcmFlaggedStars = connectionTypes.Output(
145 doc=
"Catalog of flagged stars for fgcm calibration from current fit cycle",
146 name=
"fgcmFlaggedStars{cycleNumber}",
147 storageClass=
"Catalog",
148 dimensions=(
"instrument",),
151 fgcmZeropoints = connectionTypes.Output(
152 doc=
"Catalog of fgcm zeropoint data from current fit cycle",
153 name=
"fgcmZeropoints{cycleNumber}",
154 storageClass=
"Catalog",
155 dimensions=(
"instrument",),
158 fgcmAtmosphereParameters = connectionTypes.Output(
159 doc=
"Catalog of atmospheric fit parameters from current fit cycle",
160 name=
"fgcmAtmosphereParameters{cycleNumber}",
161 storageClass=
"Catalog",
162 dimensions=(
"instrument",),
165 fgcmStandardStars = connectionTypes.Output(
166 doc=
"Catalog of standard star magnitudes from current fit cycle",
167 name=
"fgcmStandardStars{cycleNumber}",
168 storageClass=
"SimpleCatalog",
169 dimensions=(
"instrument",),
172 def __init__(self, *, config=None):
173 super().__init__(config=config)
175 if not config.doReferenceCalibration:
176 self.prerequisiteInputs.remove(
"fgcmReferenceStars")
178 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
179 raise ValueError(
"cycleNumber must be of integer format")
180 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
181 raise ValueError(
"previousCycleNumber must be of integer format")
182 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
183 raise ValueError(
"previousCycleNumber must be 1 less than cycleNumber")
185 if int(config.connections.cycleNumber) == 0:
186 self.prerequisiteInputs.remove(
"fgcmFlaggedStarsInput")
187 self.prerequisiteInputs.remove(
"fgcmFitParametersInput")
189 if not self.config.isFinalCycle
and not self.config.outputStandardsBeforeFinalCycle:
190 self.outputs.remove(
"fgcmStandardStars")
192 if not self.config.isFinalCycle
and not self.config.outputZeropointsBeforeFinalCycle:
193 self.outputs.remove(
"fgcmZeropoints")
194 self.outputs.remove(
"fgcmAtmosphereParameters")
197 class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
198 pipelineConnections=FgcmFitCycleConnections):
199 """Config for FgcmFitCycle"""
201 bands = pexConfig.ListField(
202 doc=
"Bands to run calibration",
206 fitFlag = pexConfig.ListField(
207 doc=(
"Flag for which bands are directly constrained in the FGCM fit. "
208 "Bands set to 0 will have the atmosphere constrained from observations "
209 "in other bands on the same night. Must be same length as config.bands, "
210 "and matched band-by-band."),
214 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
215 "It will be removed after v20. Use fitBands instead."),
217 fitBands = pexConfig.ListField(
218 doc=(
"Bands to use in atmospheric fit. The bands not listed here will have "
219 "the atmosphere constrained from the 'fitBands' on the same night. "
220 "Must be a subset of `config.bands`"),
224 requiredFlag = pexConfig.ListField(
225 doc=(
"Flag for which bands are required for a star to be considered a calibration "
226 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
227 "be same length as config.bands, and matched band-by-band."),
231 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
232 "It will be removed after v20. Use requiredBands instead."),
234 requiredBands = pexConfig.ListField(
235 doc=(
"Bands that are required for a star to be considered a calibration star. "
236 "Must be a subset of `config.bands`"),
240 filterMap = pexConfig.DictField(
241 doc=
"Mapping from 'filterName' to band.",
245 deprecated=(
"This field is no longer used, and has been deprecated by "
246 "DM-28088. It will be removed after v22. Use "
247 "physicalFilterMap instead.")
252 physicalFilterMap = pexConfig.DictField(
253 doc=
"Mapping from 'physicalFilter' to band.",
258 doReferenceCalibration = pexConfig.Field(
259 doc=
"Use reference catalog as additional constraint on calibration",
263 refStarSnMin = pexConfig.Field(
264 doc=
"Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
268 refStarOutlierNSig = pexConfig.Field(
269 doc=(
"Number of sigma compared to average mag for reference star to be considered an outlier. "
270 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
274 applyRefStarColorCuts = pexConfig.Field(
275 doc=
"Apply color cuts to reference stars?",
279 nCore = pexConfig.Field(
280 doc=
"Number of cores to use",
284 nStarPerRun = pexConfig.Field(
285 doc=
"Number of stars to run in each chunk",
289 nExpPerRun = pexConfig.Field(
290 doc=
"Number of exposures to run in each chunk",
294 reserveFraction = pexConfig.Field(
295 doc=
"Fraction of stars to reserve for testing",
299 freezeStdAtmosphere = pexConfig.Field(
300 doc=
"Freeze atmosphere parameters to standard (for testing)",
304 precomputeSuperStarInitialCycle = pexConfig.Field(
305 doc=
"Precompute superstar flat for initial cycle",
309 superStarSubCcd = pexConfig.Field(
310 doc=
"Compute superstar flat on sub-ccd scale",
314 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
315 "It will be removed after v20. Use superStarSubCcdDict instead."),
317 superStarSubCcdDict = pexConfig.DictField(
318 doc=(
"Per-band specification on whether to compute superstar flat on sub-ccd scale. "
319 "Must have one entry per band."),
324 superStarSubCcdChebyshevOrder = pexConfig.Field(
325 doc=(
"Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
326 "Global default is first-order polynomials, and should be overridden "
327 "on a camera-by-camera basis depending on the ISR."),
331 superStarSubCcdTriangular = pexConfig.Field(
332 doc=(
"Should the sub-ccd superstar chebyshev matrix be triangular to "
333 "suppress high-order cross terms?"),
337 superStarSigmaClip = pexConfig.Field(
338 doc=
"Number of sigma to clip outliers when selecting for superstar flats",
342 focalPlaneSigmaClip = pexConfig.Field(
343 doc=
"Number of sigma to clip outliers per focal-plane.",
347 ccdGraySubCcd = pexConfig.Field(
348 doc=
"Compute CCD gray terms on sub-ccd scale",
352 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
353 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
355 ccdGraySubCcdDict = pexConfig.DictField(
356 doc=(
"Per-band specification on whether to compute achromatic per-ccd residual "
357 "('ccd gray') on a sub-ccd scale."),
362 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
363 doc=
"Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
367 ccdGraySubCcdTriangular = pexConfig.Field(
368 doc=(
"Should the sub-ccd gray chebyshev matrix be triangular to "
369 "suppress high-order cross terms?"),
373 ccdGrayFocalPlaneDict = pexConfig.DictField(
374 doc=(
"Per-band specification on whether to compute focal-plane residual "
375 "('ccd gray') corrections."),
380 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
381 doc=(
"Minimum number of 'good' CCDs required to perform focal-plane "
382 "gray corrections. If there are fewer good CCDs then the gray "
383 "correction is computed per-ccd."),
387 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
388 doc=
"Order of the 2D chebyshev polynomials for focal plane fit.",
392 cycleNumber = pexConfig.Field(
393 doc=(
"FGCM fit cycle number. This is automatically incremented after each run "
394 "and stage of outlier rejection. See cookbook for details."),
398 isFinalCycle = pexConfig.Field(
399 doc=(
"Is this the final cycle of the fitting? Will automatically compute final "
400 "selection of stars and photometric exposures, and will output zeropoints "
401 "and standard stars for use in fgcmOutputProducts"),
405 maxIterBeforeFinalCycle = pexConfig.Field(
406 doc=(
"Maximum fit iterations, prior to final cycle. The number of iterations "
407 "will always be 0 in the final cycle for cleanup and final selection."),
411 deltaMagBkgOffsetPercentile = pexConfig.Field(
412 doc=(
"Percentile brightest stars on a visit/ccd to use to compute net "
413 "offset from local background subtraction."),
417 deltaMagBkgPerCcd = pexConfig.Field(
418 doc=(
"Compute net offset from local background subtraction per-ccd? "
419 "Otherwise, use computation per visit."),
423 utBoundary = pexConfig.Field(
424 doc=
"Boundary (in UTC) from day-to-day",
428 washMjds = pexConfig.ListField(
429 doc=
"Mirror wash MJDs",
433 epochMjds = pexConfig.ListField(
434 doc=
"Epoch boundaries in MJD",
438 minObsPerBand = pexConfig.Field(
439 doc=
"Minimum good observations per band",
445 latitude = pexConfig.Field(
446 doc=
"Observatory latitude",
450 brightObsGrayMax = pexConfig.Field(
451 doc=
"Maximum gray extinction to be considered bright observation",
455 minStarPerCcd = pexConfig.Field(
456 doc=(
"Minimum number of good stars per CCD to be used in calibration fit. "
457 "CCDs with fewer stars will have their calibration estimated from other "
458 "CCDs in the same visit, with zeropoint error increased accordingly."),
462 minCcdPerExp = pexConfig.Field(
463 doc=(
"Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
464 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
468 maxCcdGrayErr = pexConfig.Field(
469 doc=
"Maximum error on CCD gray offset to be considered photometric",
473 minStarPerExp = pexConfig.Field(
474 doc=(
"Minimum number of good stars per exposure/visit to be used in calibration fit. "
475 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
479 minExpPerNight = pexConfig.Field(
480 doc=
"Minimum number of good exposures/visits to consider a partly photometric night",
484 expGrayInitialCut = pexConfig.Field(
485 doc=(
"Maximum exposure/visit gray value for initial selection of possible photometric "
490 expGrayPhotometricCut = pexConfig.ListField(
491 doc=(
"Maximum (negative) exposure gray for a visit to be considered photometric. "
492 "Must be same length as config.bands, and matched band-by-band."),
496 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
497 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
499 expGrayPhotometricCutDict = pexConfig.DictField(
500 doc=(
"Per-band specification on maximum (negative) achromatic exposure residual "
501 "('gray term') for a visit to be considered photometric. Must have one "
502 "entry per band. Broad-band filters should be -0.05."),
507 expGrayHighCut = pexConfig.ListField(
508 doc=(
"Maximum (positive) exposure gray for a visit to be considered photometric. "
509 "Must be same length as config.bands, and matched band-by-band."),
513 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
514 "It will be removed after v20. Use expGrayHighCutDict instead."),
516 expGrayHighCutDict = pexConfig.DictField(
517 doc=(
"Per-band specification on maximum (positive) achromatic exposure residual "
518 "('gray term') for a visit to be considered photometric. Must have one "
519 "entry per band. Broad-band filters should be 0.2."),
524 expGrayRecoverCut = pexConfig.Field(
525 doc=(
"Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
526 "Visits with more gray extinction will only get CCD zeropoints if there are "
527 "sufficient star observations (minStarPerCcd) on that CCD."),
531 expVarGrayPhotometricCut = pexConfig.Field(
532 doc=
"Maximum exposure variance to be considered possibly photometric",
536 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
537 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
539 expVarGrayPhotometricCutDict = pexConfig.DictField(
540 doc=(
"Per-band specification on maximum exposure variance to be considered possibly "
541 "photometric. Must have one entry per band. Broad-band filters should be "
547 expGrayErrRecoverCut = pexConfig.Field(
548 doc=(
"Maximum exposure gray error to be able to recover bad ccds via interpolation. "
549 "Visits with more gray variance will only get CCD zeropoints if there are "
550 "sufficient star observations (minStarPerCcd) on that CCD."),
554 aperCorrFitNBins = pexConfig.Field(
555 doc=(
"Number of aperture bins used in aperture correction fit. When set to 0"
556 "no fit will be performed, and the config.aperCorrInputSlopes will be "
557 "used if available."),
561 aperCorrInputSlopes = pexConfig.ListField(
562 doc=(
"Aperture correction input slope parameters. These are used on the first "
563 "fit iteration, and aperture correction parameters will be updated from "
564 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
565 "when there is insufficient data to fit the parameters (e.g. tract mode). "
566 "If set, must be same length as config.bands, and matched band-by-band."),
570 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
571 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
573 aperCorrInputSlopeDict = pexConfig.DictField(
574 doc=(
"Per-band specification of aperture correction input slope parameters. These "
575 "are used on the first fit iteration, and aperture correction parameters will "
576 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
577 "to set this when there is insufficient data to fit the parameters (e.g. "
583 sedFudgeFactors = pexConfig.ListField(
584 doc=(
"Fudge factors for computing linear SED from colors. Must be same length as "
585 "config.bands, and matched band-by-band."),
589 deprecated=(
"This field has been deprecated and will be removed after v20. "
590 "Please use sedSlopeTermMap and sedSlopeMap."),
592 sedboundaryterms = pexConfig.ConfigField(
593 doc=
"Mapping from bands to SED boundary term names used is sedterms.",
594 dtype=SedboundarytermDict,
596 sedterms = pexConfig.ConfigField(
597 doc=
"Mapping from terms to bands for fgcm linear SED approximations.",
600 sigFgcmMaxErr = pexConfig.Field(
601 doc=
"Maximum mag error for fitting sigma_FGCM",
605 sigFgcmMaxEGray = pexConfig.ListField(
606 doc=(
"Maximum (absolute) gray value for observation in sigma_FGCM. "
607 "May be 1 element (same for all bands) or the same length as config.bands."),
611 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
612 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
614 sigFgcmMaxEGrayDict = pexConfig.DictField(
615 doc=(
"Per-band specification for maximum (absolute) achromatic residual (gray value) "
616 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
622 ccdGrayMaxStarErr = pexConfig.Field(
623 doc=(
"Maximum error on a star observation to use in ccd gray (achromatic residual) "
628 approxThroughput = pexConfig.ListField(
629 doc=(
"Approximate overall throughput at start of calibration observations. "
630 "May be 1 element (same for all bands) or the same length as config.bands."),
634 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
635 "It will be removed after v20. Use approxThroughputDict instead."),
637 approxThroughputDict = pexConfig.DictField(
638 doc=(
"Per-band specification of the approximate overall throughput at the start of "
639 "calibration observations. Must have one entry per band. Typically should "
645 sigmaCalRange = pexConfig.ListField(
646 doc=
"Allowed range for systematic error floor estimation",
648 default=(0.001, 0.003),
650 sigmaCalFitPercentile = pexConfig.ListField(
651 doc=
"Magnitude percentile range to fit systematic error floor",
653 default=(0.05, 0.15),
655 sigmaCalPlotPercentile = pexConfig.ListField(
656 doc=
"Magnitude percentile range to plot systematic error floor",
658 default=(0.05, 0.95),
660 sigma0Phot = pexConfig.Field(
661 doc=
"Systematic error floor for all zeropoints",
665 mapLongitudeRef = pexConfig.Field(
666 doc=
"Reference longitude for plotting maps",
670 mapNSide = pexConfig.Field(
671 doc=
"Healpix nside for plotting maps",
675 outfileBase = pexConfig.Field(
676 doc=
"Filename start for plot output files",
680 starColorCuts = pexConfig.ListField(
681 doc=
"Encoded star-color cuts (to be cleaned up)",
683 default=(
"NO_DATA",),
685 colorSplitIndices = pexConfig.ListField(
686 doc=
"Band indices to use to split stars by color",
690 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
691 "It will be removed after v20. Use colorSplitBands instead."),
693 colorSplitBands = pexConfig.ListField(
694 doc=
"Band names to use to split stars by color. Must have 2 entries.",
699 modelMagErrors = pexConfig.Field(
700 doc=
"Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
704 useQuadraticPwv = pexConfig.Field(
705 doc=
"Model PWV with a quadratic term for variation through the night?",
709 instrumentParsPerBand = pexConfig.Field(
710 doc=(
"Model instrumental parameters per band? "
711 "Otherwise, instrumental parameters (QE changes with time) are "
712 "shared among all bands."),
716 instrumentSlopeMinDeltaT = pexConfig.Field(
717 doc=(
"Minimum time change (in days) between observations to use in constraining "
718 "instrument slope."),
722 fitMirrorChromaticity = pexConfig.Field(
723 doc=
"Fit (intraband) mirror chromatic term?",
727 coatingMjds = pexConfig.ListField(
728 doc=
"Mirror coating dates in MJD",
732 outputStandardsBeforeFinalCycle = pexConfig.Field(
733 doc=
"Output standard stars prior to final cycle? Used in debugging.",
737 outputZeropointsBeforeFinalCycle = pexConfig.Field(
738 doc=
"Output standard stars prior to final cycle? Used in debugging.",
742 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
743 doc=(
"Use star repeatability (instead of exposures) for computing photometric "
744 "cuts? Recommended for tract mode or bands with few exposures. "
745 "May be 1 element (same for all bands) or the same length as config.bands."),
749 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
750 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
752 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
753 doc=(
"Per-band specification on whether to use star repeatability (instead of exposures) "
754 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
759 autoPhotometricCutNSig = pexConfig.Field(
760 doc=(
"Number of sigma for automatic computation of (low) photometric cut. "
761 "Cut is based on exposure gray width (per band), unless "
762 "useRepeatabilityForExpGrayCuts is set, in which case the star "
763 "repeatability is used (also per band)."),
767 autoHighCutNSig = pexConfig.Field(
768 doc=(
"Number of sigma for automatic computation of (high) outlier cut. "
769 "Cut is based on exposure gray width (per band), unless "
770 "useRepeatabilityForExpGrayCuts is set, in which case the star "
771 "repeatability is used (also per band)."),
775 quietMode = pexConfig.Field(
776 doc=
"Be less verbose with logging.",
780 randomSeed = pexConfig.Field(
781 doc=
"Random seed for fgcm for consistency in tests.",
790 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
791 msg =
"cycleNumber in template must be connections.previousCycleNumber + 1"
792 raise RuntimeError(msg)
793 if self.connections.cycleNumber != str(self.cycleNumber):
794 msg =
"cycleNumber in template must be equal to connections.cycleNumber"
795 raise RuntimeError(msg)
797 for band
in self.fitBands:
798 if band
not in self.bands:
799 msg =
'fitBand %s not in bands' % (band)
800 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
801 for band
in self.requiredBands:
802 if band
not in self.bands:
803 msg =
'requiredBand %s not in bands' % (band)
804 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
805 for band
in self.colorSplitBands:
806 if band
not in self.bands:
807 msg =
'colorSplitBand %s not in bands' % (band)
808 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
809 for band
in self.bands:
810 if band
not in self.superStarSubCcdDict:
811 msg =
'band %s not in superStarSubCcdDict' % (band)
812 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
814 if band
not in self.ccdGraySubCcdDict:
815 msg =
'band %s not in ccdGraySubCcdDict' % (band)
816 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
818 if band
not in self.expGrayPhotometricCutDict:
819 msg =
'band %s not in expGrayPhotometricCutDict' % (band)
820 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
822 if band
not in self.expGrayHighCutDict:
823 msg =
'band %s not in expGrayHighCutDict' % (band)
824 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
826 if band
not in self.expVarGrayPhotometricCutDict:
827 msg =
'band %s not in expVarGrayPhotometricCutDict' % (band)
828 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
830 if band
not in self.sigFgcmMaxEGrayDict:
831 msg =
'band %s not in sigFgcmMaxEGrayDict' % (band)
832 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
834 if band
not in self.approxThroughputDict:
835 msg =
'band %s not in approxThroughputDict' % (band)
836 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
838 if band
not in self.useRepeatabilityForExpGrayCutsDict:
839 msg =
'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
840 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
844 class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
845 """Subclass of TaskRunner for fgcmFitCycleTask
847 fgcmFitCycleTask.run() takes one argument, the butler, and uses
848 stars and visits previously extracted from dataRefs by
850 This Runner does not perform any dataRef parallelization, but the FGCM
851 code called by the Task uses python multiprocessing (see the "ncores"
856 def getTargetList(parsedCmd):
858 Return a list with one element, the butler.
860 return [parsedCmd.butler]
862 def __call__(self, butler):
866 butler: `lsst.daf.persistence.Butler`
870 exitStatus: `list` with `pipeBase.Struct`
871 exitStatus (0: success; 1: failure)
874 task = self.TaskClass(config=self.config, log=self.log)
878 task.runDataRef(butler)
881 task.runDataRef(butler)
882 except Exception
as e:
884 task.log.fatal(
"Failed: %s" % e)
885 if not isinstance(e, pipeBase.TaskError):
886 traceback.print_exc(file=sys.stderr)
888 task.writeMetadata(butler)
891 return [pipeBase.Struct(exitStatus=exitStatus)]
893 def run(self, parsedCmd):
895 Run the task, with no multiprocessing
899 parsedCmd: ArgumentParser parsed command line
904 if self.precall(parsedCmd):
905 targetList = self.getTargetList(parsedCmd)
907 resultList = self(targetList[0])
912 class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
914 Run Single fit cycle for FGCM global calibration
917 ConfigClass = FgcmFitCycleConfig
918 RunnerClass = FgcmFitCycleRunner
919 _DefaultName =
"fgcmFitCycle"
921 def __init__(self, butler=None, initInputs=None, **kwargs):
922 super().__init__(**kwargs)
925 def _getMetadataName(self):
928 def runQuantum(self, butlerQC, inputRefs, outputRefs):
929 camera = butlerQC.get(inputRefs.camera)
933 dataRefDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
934 dataRefDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
935 dataRefDict[
'fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
936 dataRefDict[
'fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
937 dataRefDict[
'fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
938 if self.config.doReferenceCalibration:
939 dataRefDict[
'fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
940 if self.config.cycleNumber > 0:
941 dataRefDict[
'fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
942 dataRefDict[
'fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
944 fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
946 butlerQC.put(fgcmDatasetDict[
'fgcmFitParameters'], outputRefs.fgcmFitParameters)
947 butlerQC.put(fgcmDatasetDict[
'fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
948 if self.outputZeropoints:
949 butlerQC.put(fgcmDatasetDict[
'fgcmZeropoints'], outputRefs.fgcmZeropoints)
950 butlerQC.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
951 if self.outputStandards:
952 butlerQC.put(fgcmDatasetDict[
'fgcmStandardStars'], outputRefs.fgcmStandardStars)
955 def runDataRef(self, butler):
957 Run a single fit cycle for FGCM
961 butler: `lsst.daf.persistence.Butler`
963 self._checkDatasetsExist(butler)
966 dataRefDict[
'fgcmLookUpTable'] = butler.dataRef(
'fgcmLookUpTable')
967 dataRefDict[
'fgcmVisitCatalog'] = butler.dataRef(
'fgcmVisitCatalog')
968 dataRefDict[
'fgcmStarObservations'] = butler.dataRef(
'fgcmStarObservations')
969 dataRefDict[
'fgcmStarIds'] = butler.dataRef(
'fgcmStarIds')
970 dataRefDict[
'fgcmStarIndices'] = butler.dataRef(
'fgcmStarIndices')
971 if self.config.doReferenceCalibration:
972 dataRefDict[
'fgcmReferenceStars'] = butler.dataRef(
'fgcmReferenceStars')
973 if self.config.cycleNumber > 0:
974 lastCycle = self.config.cycleNumber - 1
975 dataRefDict[
'fgcmFlaggedStars'] = butler.dataRef(
'fgcmFlaggedStars',
977 dataRefDict[
'fgcmFitParameters'] = butler.dataRef(
'fgcmFitParameters',
980 camera = butler.get(
'camera')
981 fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
983 butler.put(fgcmDatasetDict[
'fgcmFitParameters'],
'fgcmFitParameters',
984 fgcmcycle=self.config.cycleNumber)
985 butler.put(fgcmDatasetDict[
'fgcmFlaggedStars'],
'fgcmFlaggedStars',
986 fgcmcycle=self.config.cycleNumber)
987 if self.outputZeropoints:
988 butler.put(fgcmDatasetDict[
'fgcmZeropoints'],
'fgcmZeropoints',
989 fgcmcycle=self.config.cycleNumber)
990 butler.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'],
'fgcmAtmosphereParameters',
991 fgcmcycle=self.config.cycleNumber)
992 if self.outputStandards:
993 butler.put(fgcmDatasetDict[
'fgcmStandardStars'],
'fgcmStandardStars',
994 fgcmcycle=self.config.cycleNumber)
996 def writeConfig(self, butler, clobber=False, doBackup=True):
997 """Write the configuration used for processing the data, or check that an existing
998 one is equal to the new one if present. This is an override of the regular
999 version from pipe_base that knows about fgcmcycle.
1003 butler : `lsst.daf.persistence.Butler`
1004 Data butler used to write the config. The config is written to dataset type
1005 `CmdLineTask._getConfigName`.
1006 clobber : `bool`, optional
1007 A boolean flag that controls what happens if a config already has been saved:
1008 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1009 - `False`: raise `TaskError` if this config does not match the existing config.
1010 doBackup : `bool`, optional
1011 Set to `True` to backup the config files if clobbering.
1013 configName = self._getConfigName()
1014 if configName
is None:
1017 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1018 elif butler.datasetExists(configName, write=
True, fgcmcycle=self.config.cycleNumber):
1021 oldConfig = butler.get(configName, immediate=
True, fgcmcycle=self.config.cycleNumber)
1022 except Exception
as exc:
1023 raise type(exc)(
"Unable to read stored config file %s (%s); consider using --clobber-config" %
1026 def logConfigMismatch(msg):
1027 self.log.fatal(
"Comparing configuration: %s", msg)
1029 if not self.config.compare(oldConfig, shortcut=
False, output=logConfigMismatch):
1030 raise pipeBase.TaskError(
1031 f
"Config does not match existing task config {configName!r} on disk; tasks configurations"
1032 " must be consistent within the same output repo (override with --clobber-config)")
1034 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1036 def _fgcmFitCycle(self, camera, dataRefDict):
1042 camera : `lsst.afw.cameraGeom.Camera`
1043 dataRefDict : `dict`
1044 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1045 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1046 dataRef dictionary with keys:
1048 ``"fgcmLookUpTable"``
1049 dataRef for the FGCM look-up table.
1050 ``"fgcmVisitCatalog"``
1051 dataRef for visit summary catalog.
1052 ``"fgcmStarObservations"``
1053 dataRef for star observation catalog.
1055 dataRef for star id catalog.
1056 ``"fgcmStarIndices"``
1057 dataRef for star index catalog.
1058 ``"fgcmReferenceStars"``
1059 dataRef for matched reference star catalog.
1060 ``"fgcmFlaggedStars"``
1061 dataRef for flagged star catalog.
1062 ``"fgcmFitParameters"``
1063 dataRef for fit parameter catalog.
1067 fgcmDatasetDict : `dict`
1068 Dictionary of datasets to persist.
1071 self.maxIter = self.config.maxIterBeforeFinalCycle
1072 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
1073 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
1074 self.resetFitParameters =
True
1076 if self.config.isFinalCycle:
1081 self.outputStandards =
True
1082 self.outputZeropoints =
True
1083 self.resetFitParameters =
False
1086 self.maxIter, self.resetFitParameters,
1087 self.outputZeropoints)
1089 lutCat = dataRefDict[
'fgcmLookUpTable'].get()
1091 dict(self.config.physicalFilterMap))
1095 visitCat = dataRefDict[
'fgcmVisitCatalog'].get()
1103 noFitsDict = {
'lutIndex': lutIndexVals,
1105 'expInfo': fgcmExpInfo,
1106 'ccdOffsets': ccdOffsets}
1109 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
1110 noFitsDict=noFitsDict, noOutput=
True)
1113 if (fgcmFitCycle.initialCycle):
1115 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1119 inParInfo, inParams, inSuperStar = self._loadParameters(dataRefDict[
'fgcmFitParameters'])
1120 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1127 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1129 starObs = dataRefDict[
'fgcmStarObservations'].get()
1130 starIds = dataRefDict[
'fgcmStarIds'].get()
1131 starIndices = dataRefDict[
'fgcmStarIndices'].get()
1134 if 'fgcmFlaggedStars' in dataRefDict:
1135 flaggedStars = dataRefDict[
'fgcmFlaggedStars'].get()
1136 flagId = flaggedStars[
'objId'][:]
1137 flagFlag = flaggedStars[
'objFlag'][:]
1143 if self.config.doReferenceCalibration:
1144 refStars = dataRefDict[
'fgcmReferenceStars'].get()
1148 self.config.physicalFilterMap)
1149 refId = refStars[
'fgcm_id'][:]
1159 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'], starObs[
'visit'][starIndices[
'obsIndex']])
1171 conv = starObs[0][
'ra'].asDegrees() / float(starObs[0][
'ra'])
1173 fgcmStars.loadStars(fgcmPars,
1174 starObs[
'visit'][starIndices[
'obsIndex']],
1175 starObs[
'ccd'][starIndices[
'obsIndex']],
1176 starObs[
'ra'][starIndices[
'obsIndex']] * conv,
1177 starObs[
'dec'][starIndices[
'obsIndex']] * conv,
1178 starObs[
'instMag'][starIndices[
'obsIndex']],
1179 starObs[
'instMagErr'][starIndices[
'obsIndex']],
1180 fgcmExpInfo[
'FILTERNAME'][visitIndex],
1181 starIds[
'fgcm_id'][:],
1184 starIds[
'obsArrIndex'][:],
1186 obsX=starObs[
'x'][starIndices[
'obsIndex']],
1187 obsY=starObs[
'y'][starIndices[
'obsIndex']],
1188 obsDeltaMagBkg=starObs[
'deltaMagBkg'][starIndices[
'obsIndex']],
1189 psfCandidate=starObs[
'psf_candidate'][starIndices[
'obsIndex']],
1192 refMagErr=refMagErr,
1210 fgcmFitCycle.setLUT(fgcmLut)
1211 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1212 fgcmFitCycle.setPars(fgcmPars)
1215 fgcmFitCycle.finishSetup()
1224 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1229 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i])
for
1230 i, b
in enumerate(self.config.bands)}
1231 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i])
for
1232 i, band
in enumerate(self.config.bands)}
1234 outConfig = copy.copy(self.config)
1235 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
1236 precomputeSuperStarInitialCycle=
False,
1237 freezeStdAtmosphere=
False,
1238 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1239 expGrayHighCutDict=updatedHighCutDict)
1241 outConfig.connections.update(previousCycleNumber=str(self.config.cycleNumber),
1242 cycleNumber=str(self.config.cycleNumber + 1))
1244 configFileName =
'%s_cycle%02d_config.py' % (outConfig.outfileBase,
1245 outConfig.cycleNumber)
1246 outConfig.save(configFileName)
1248 if self.config.isFinalCycle == 1:
1250 self.log.info(
"Everything is in place to run fgcmOutputProducts.py")
1252 self.log.info(
"Saved config for next cycle to %s" % (configFileName))
1253 self.log.info(
"Be sure to look at:")
1254 self.log.info(
" config.expGrayPhotometricCut")
1255 self.log.info(
" config.expGrayHighCut")
1256 self.log.info(
"If you are satisfied with the fit, please set:")
1257 self.log.info(
" config.isFinalCycle = True")
1259 return fgcmDatasetDict
1261 def _checkDatasetsExist(self, butler):
1263 Check if necessary datasets exist to run fgcmFitCycle
1267 butler: `lsst.daf.persistence.Butler`
1272 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1273 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1274 If cycleNumber > 0, then also checks for fgcmFitParameters,
1278 if not butler.datasetExists(
'fgcmVisitCatalog'):
1279 raise RuntimeError(
"Could not find fgcmVisitCatalog in repo!")
1280 if not butler.datasetExists(
'fgcmStarObservations'):
1281 raise RuntimeError(
"Could not find fgcmStarObservations in repo!")
1282 if not butler.datasetExists(
'fgcmStarIds'):
1283 raise RuntimeError(
"Could not find fgcmStarIds in repo!")
1284 if not butler.datasetExists(
'fgcmStarIndices'):
1285 raise RuntimeError(
"Could not find fgcmStarIndices in repo!")
1286 if not butler.datasetExists(
'fgcmLookUpTable'):
1287 raise RuntimeError(
"Could not find fgcmLookUpTable in repo!")
1290 if (self.config.cycleNumber > 0):
1291 if not butler.datasetExists(
'fgcmFitParameters',
1292 fgcmcycle=self.config.cycleNumber-1):
1293 raise RuntimeError(
"Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1294 (self.config.cycleNumber-1))
1295 if not butler.datasetExists(
'fgcmFlaggedStars',
1296 fgcmcycle=self.config.cycleNumber-1):
1297 raise RuntimeError(
"Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1298 (self.config.cycleNumber-1))
1301 if self.config.doReferenceCalibration:
1302 if not butler.datasetExists(
'fgcmReferenceStars'):
1303 raise RuntimeError(
"Could not find fgcmReferenceStars in repo, and "
1304 "doReferenceCalibration is True.")
1306 def _loadParameters(self, parDataRef):
1308 Load FGCM parameters from a previous fit cycle
1312 parDataRef : `lsst.daf.persistence.ButlerDataRef` or `lsst.daf.butler.DeferredDatasetHandle`
1313 dataRef for previous fit parameter catalog.
1317 inParInfo: `numpy.ndarray`
1318 Numpy array parameter information formatted for input to fgcm
1319 inParameters: `numpy.ndarray`
1320 Numpy array parameter values formatted for input to fgcm
1321 inSuperStar: `numpy.array`
1322 Superstar flat formatted for input to fgcm
1325 parCat = parDataRef.get()
1327 parLutFilterNames = np.array(parCat[0][
'lutFilterNames'].split(
','))
1328 parFitBands = np.array(parCat[0][
'fitBands'].split(
','))
1330 inParInfo = np.zeros(1, dtype=[(
'NCCD',
'i4'),
1331 (
'LUTFILTERNAMES', parLutFilterNames.dtype.str,
1332 (parLutFilterNames.size, )),
1333 (
'FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1334 (
'LNTAUUNIT',
'f8'),
1335 (
'LNTAUSLOPEUNIT',
'f8'),
1336 (
'ALPHAUNIT',
'f8'),
1337 (
'LNPWVUNIT',
'f8'),
1338 (
'LNPWVSLOPEUNIT',
'f8'),
1339 (
'LNPWVQUADRATICUNIT',
'f8'),
1340 (
'LNPWVGLOBALUNIT',
'f8'),
1342 (
'QESYSUNIT',
'f8'),
1343 (
'FILTEROFFSETUNIT',
'f8'),
1344 (
'HASEXTERNALPWV',
'i2'),
1345 (
'HASEXTERNALTAU',
'i2')])
1346 inParInfo[
'NCCD'] = parCat[
'nCcd']
1347 inParInfo[
'LUTFILTERNAMES'][:] = parLutFilterNames
1348 inParInfo[
'FITBANDS'][:] = parFitBands
1349 inParInfo[
'HASEXTERNALPWV'] = parCat[
'hasExternalPwv']
1350 inParInfo[
'HASEXTERNALTAU'] = parCat[
'hasExternalTau']
1352 inParams = np.zeros(1, dtype=[(
'PARALPHA',
'f8', (parCat[
'parAlpha'].size, )),
1353 (
'PARO3',
'f8', (parCat[
'parO3'].size, )),
1354 (
'PARLNTAUINTERCEPT',
'f8',
1355 (parCat[
'parLnTauIntercept'].size, )),
1356 (
'PARLNTAUSLOPE',
'f8',
1357 (parCat[
'parLnTauSlope'].size, )),
1358 (
'PARLNPWVINTERCEPT',
'f8',
1359 (parCat[
'parLnPwvIntercept'].size, )),
1360 (
'PARLNPWVSLOPE',
'f8',
1361 (parCat[
'parLnPwvSlope'].size, )),
1362 (
'PARLNPWVQUADRATIC',
'f8',
1363 (parCat[
'parLnPwvQuadratic'].size, )),
1364 (
'PARQESYSINTERCEPT',
'f8',
1365 (parCat[
'parQeSysIntercept'].size, )),
1366 (
'COMPQESYSSLOPE',
'f8',
1367 (parCat[
'compQeSysSlope'].size, )),
1368 (
'PARFILTEROFFSET',
'f8',
1369 (parCat[
'parFilterOffset'].size, )),
1370 (
'PARFILTEROFFSETFITFLAG',
'i2',
1371 (parCat[
'parFilterOffsetFitFlag'].size, )),
1372 (
'PARRETRIEVEDLNPWVSCALE',
'f8'),
1373 (
'PARRETRIEVEDLNPWVOFFSET',
'f8'),
1374 (
'PARRETRIEVEDLNPWVNIGHTLYOFFSET',
'f8',
1375 (parCat[
'parRetrievedLnPwvNightlyOffset'].size, )),
1376 (
'COMPABSTHROUGHPUT',
'f8',
1377 (parCat[
'compAbsThroughput'].size, )),
1378 (
'COMPREFOFFSET',
'f8',
1379 (parCat[
'compRefOffset'].size, )),
1380 (
'COMPREFSIGMA',
'f8',
1381 (parCat[
'compRefSigma'].size, )),
1382 (
'COMPMIRRORCHROMATICITY',
'f8',
1383 (parCat[
'compMirrorChromaticity'].size, )),
1384 (
'MIRRORCHROMATICITYPIVOT',
'f8',
1385 (parCat[
'mirrorChromaticityPivot'].size, )),
1386 (
'COMPMEDIANSEDSLOPE',
'f8',
1387 (parCat[
'compMedianSedSlope'].size, )),
1388 (
'COMPAPERCORRPIVOT',
'f8',
1389 (parCat[
'compAperCorrPivot'].size, )),
1390 (
'COMPAPERCORRSLOPE',
'f8',
1391 (parCat[
'compAperCorrSlope'].size, )),
1392 (
'COMPAPERCORRSLOPEERR',
'f8',
1393 (parCat[
'compAperCorrSlopeErr'].size, )),
1394 (
'COMPAPERCORRRANGE',
'f8',
1395 (parCat[
'compAperCorrRange'].size, )),
1396 (
'COMPMODELERREXPTIMEPIVOT',
'f8',
1397 (parCat[
'compModelErrExptimePivot'].size, )),
1398 (
'COMPMODELERRFWHMPIVOT',
'f8',
1399 (parCat[
'compModelErrFwhmPivot'].size, )),
1400 (
'COMPMODELERRSKYPIVOT',
'f8',
1401 (parCat[
'compModelErrSkyPivot'].size, )),
1402 (
'COMPMODELERRPARS',
'f8',
1403 (parCat[
'compModelErrPars'].size, )),
1404 (
'COMPEXPGRAY',
'f8',
1405 (parCat[
'compExpGray'].size, )),
1406 (
'COMPVARGRAY',
'f8',
1407 (parCat[
'compVarGray'].size, )),
1408 (
'COMPEXPDELTAMAGBKG',
'f8',
1409 (parCat[
'compExpDeltaMagBkg'].size, )),
1410 (
'COMPNGOODSTARPEREXP',
'i4',
1411 (parCat[
'compNGoodStarPerExp'].size, )),
1412 (
'COMPSIGFGCM',
'f8',
1413 (parCat[
'compSigFgcm'].size, )),
1414 (
'COMPSIGMACAL',
'f8',
1415 (parCat[
'compSigmaCal'].size, )),
1416 (
'COMPRETRIEVEDLNPWV',
'f8',
1417 (parCat[
'compRetrievedLnPwv'].size, )),
1418 (
'COMPRETRIEVEDLNPWVRAW',
'f8',
1419 (parCat[
'compRetrievedLnPwvRaw'].size, )),
1420 (
'COMPRETRIEVEDLNPWVFLAG',
'i2',
1421 (parCat[
'compRetrievedLnPwvFlag'].size, )),
1422 (
'COMPRETRIEVEDTAUNIGHT',
'f8',
1423 (parCat[
'compRetrievedTauNight'].size, ))])
1425 inParams[
'PARALPHA'][:] = parCat[
'parAlpha'][0, :]
1426 inParams[
'PARO3'][:] = parCat[
'parO3'][0, :]
1427 inParams[
'PARLNTAUINTERCEPT'][:] = parCat[
'parLnTauIntercept'][0, :]
1428 inParams[
'PARLNTAUSLOPE'][:] = parCat[
'parLnTauSlope'][0, :]
1429 inParams[
'PARLNPWVINTERCEPT'][:] = parCat[
'parLnPwvIntercept'][0, :]
1430 inParams[
'PARLNPWVSLOPE'][:] = parCat[
'parLnPwvSlope'][0, :]
1431 inParams[
'PARLNPWVQUADRATIC'][:] = parCat[
'parLnPwvQuadratic'][0, :]
1432 inParams[
'PARQESYSINTERCEPT'][:] = parCat[
'parQeSysIntercept'][0, :]
1433 inParams[
'COMPQESYSSLOPE'][:] = parCat[
'compQeSysSlope'][0, :]
1434 inParams[
'PARFILTEROFFSET'][:] = parCat[
'parFilterOffset'][0, :]
1435 inParams[
'PARFILTEROFFSETFITFLAG'][:] = parCat[
'parFilterOffsetFitFlag'][0, :]
1436 inParams[
'PARRETRIEVEDLNPWVSCALE'] = parCat[
'parRetrievedLnPwvScale']
1437 inParams[
'PARRETRIEVEDLNPWVOFFSET'] = parCat[
'parRetrievedLnPwvOffset']
1438 inParams[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat[
'parRetrievedLnPwvNightlyOffset'][0, :]
1439 inParams[
'COMPABSTHROUGHPUT'][:] = parCat[
'compAbsThroughput'][0, :]
1440 inParams[
'COMPREFOFFSET'][:] = parCat[
'compRefOffset'][0, :]
1441 inParams[
'COMPREFSIGMA'][:] = parCat[
'compRefSigma'][0, :]
1442 inParams[
'COMPMIRRORCHROMATICITY'][:] = parCat[
'compMirrorChromaticity'][0, :]
1443 inParams[
'MIRRORCHROMATICITYPIVOT'][:] = parCat[
'mirrorChromaticityPivot'][0, :]
1444 inParams[
'COMPMEDIANSEDSLOPE'][:] = parCat[
'compMedianSedSlope'][0, :]
1445 inParams[
'COMPAPERCORRPIVOT'][:] = parCat[
'compAperCorrPivot'][0, :]
1446 inParams[
'COMPAPERCORRSLOPE'][:] = parCat[
'compAperCorrSlope'][0, :]
1447 inParams[
'COMPAPERCORRSLOPEERR'][:] = parCat[
'compAperCorrSlopeErr'][0, :]
1448 inParams[
'COMPAPERCORRRANGE'][:] = parCat[
'compAperCorrRange'][0, :]
1449 inParams[
'COMPMODELERREXPTIMEPIVOT'][:] = parCat[
'compModelErrExptimePivot'][0, :]
1450 inParams[
'COMPMODELERRFWHMPIVOT'][:] = parCat[
'compModelErrFwhmPivot'][0, :]
1451 inParams[
'COMPMODELERRSKYPIVOT'][:] = parCat[
'compModelErrSkyPivot'][0, :]
1452 inParams[
'COMPMODELERRPARS'][:] = parCat[
'compModelErrPars'][0, :]
1453 inParams[
'COMPEXPGRAY'][:] = parCat[
'compExpGray'][0, :]
1454 inParams[
'COMPVARGRAY'][:] = parCat[
'compVarGray'][0, :]
1455 inParams[
'COMPEXPDELTAMAGBKG'][:] = parCat[
'compExpDeltaMagBkg'][0, :]
1456 inParams[
'COMPNGOODSTARPEREXP'][:] = parCat[
'compNGoodStarPerExp'][0, :]
1457 inParams[
'COMPSIGFGCM'][:] = parCat[
'compSigFgcm'][0, :]
1458 inParams[
'COMPSIGMACAL'][:] = parCat[
'compSigmaCal'][0, :]
1459 inParams[
'COMPRETRIEVEDLNPWV'][:] = parCat[
'compRetrievedLnPwv'][0, :]
1460 inParams[
'COMPRETRIEVEDLNPWVRAW'][:] = parCat[
'compRetrievedLnPwvRaw'][0, :]
1461 inParams[
'COMPRETRIEVEDLNPWVFLAG'][:] = parCat[
'compRetrievedLnPwvFlag'][0, :]
1462 inParams[
'COMPRETRIEVEDTAUNIGHT'][:] = parCat[
'compRetrievedTauNight'][0, :]
1464 inSuperStar = np.zeros(parCat[
'superstarSize'][0, :], dtype=
'f8')
1465 inSuperStar[:, :, :, :] = parCat[
'superstar'][0, :].reshape(inSuperStar.shape)
1467 return (inParInfo, inParams, inSuperStar)
1469 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1471 Persist FGCM datasets through the butler.
1475 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1476 Fgcm Fit cycle object
1478 fgcmDatasetDict = {}
1481 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1483 parSchema = afwTable.Schema()
1486 lutFilterNameString = comma.join([n.decode(
'utf-8')
1487 for n
in parInfo[
'LUTFILTERNAMES'][0]])
1488 fitBandString = comma.join([n.decode(
'utf-8')
1489 for n
in parInfo[
'FITBANDS'][0]])
1491 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1492 lutFilterNameString, fitBandString)
1493 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1494 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1495 lutFilterNameString, fitBandString)
1497 fgcmDatasetDict[
'fgcmFitParameters'] = parCat
1502 flagStarSchema = self._makeFlagStarSchema()
1503 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1504 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1506 fgcmDatasetDict[
'fgcmFlaggedStars'] = flagStarCat
1509 if self.outputZeropoints:
1510 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
1511 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
1514 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1516 fgcmDatasetDict[
'fgcmZeropoints'] = zptCat
1521 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1523 fgcmDatasetDict[
'fgcmAtmosphereParameters'] = atmCat
1526 if self.outputStandards:
1527 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1529 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
1531 fgcmDatasetDict[
'fgcmStandardStars'] = stdCat
1533 return fgcmDatasetDict
1535 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1536 lutFilterNameString, fitBandString):
1538 Make the parameter persistence schema
1542 parInfo: `numpy.ndarray`
1543 Parameter information returned by fgcm
1544 pars: `numpy.ndarray`
1545 Parameter values returned by fgcm
1546 parSuperStarFlat: `numpy.array`
1547 Superstar flat values returned by fgcm
1548 lutFilterNameString: `str`
1549 Combined string of all the lutFilterNames
1550 fitBandString: `str`
1551 Combined string of all the fitBands
1555 parSchema: `afwTable.schema`
1558 parSchema = afwTable.Schema()
1561 parSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
1562 parSchema.addField(
'lutFilterNames', type=str, doc=
'LUT Filter names in parameter file',
1563 size=len(lutFilterNameString))
1564 parSchema.addField(
'fitBands', type=str, doc=
'Bands that were fit',
1565 size=len(fitBandString))
1566 parSchema.addField(
'lnTauUnit', type=np.float64, doc=
'Step units for ln(AOD)')
1567 parSchema.addField(
'lnTauSlopeUnit', type=np.float64,
1568 doc=
'Step units for ln(AOD) slope')
1569 parSchema.addField(
'alphaUnit', type=np.float64, doc=
'Step units for alpha')
1570 parSchema.addField(
'lnPwvUnit', type=np.float64, doc=
'Step units for ln(pwv)')
1571 parSchema.addField(
'lnPwvSlopeUnit', type=np.float64,
1572 doc=
'Step units for ln(pwv) slope')
1573 parSchema.addField(
'lnPwvQuadraticUnit', type=np.float64,
1574 doc=
'Step units for ln(pwv) quadratic term')
1575 parSchema.addField(
'lnPwvGlobalUnit', type=np.float64,
1576 doc=
'Step units for global ln(pwv) parameters')
1577 parSchema.addField(
'o3Unit', type=np.float64, doc=
'Step units for O3')
1578 parSchema.addField(
'qeSysUnit', type=np.float64, doc=
'Step units for mirror gray')
1579 parSchema.addField(
'filterOffsetUnit', type=np.float64, doc=
'Step units for filter offset')
1580 parSchema.addField(
'hasExternalPwv', type=np.int32, doc=
'Parameters fit using external pwv')
1581 parSchema.addField(
'hasExternalTau', type=np.int32, doc=
'Parameters fit using external tau')
1584 parSchema.addField(
'parAlpha', type=
'ArrayD', doc=
'Alpha parameter vector',
1585 size=pars[
'PARALPHA'].size)
1586 parSchema.addField(
'parO3', type=
'ArrayD', doc=
'O3 parameter vector',
1587 size=pars[
'PARO3'].size)
1588 parSchema.addField(
'parLnTauIntercept', type=
'ArrayD',
1589 doc=
'ln(Tau) intercept parameter vector',
1590 size=pars[
'PARLNTAUINTERCEPT'].size)
1591 parSchema.addField(
'parLnTauSlope', type=
'ArrayD',
1592 doc=
'ln(Tau) slope parameter vector',
1593 size=pars[
'PARLNTAUSLOPE'].size)
1594 parSchema.addField(
'parLnPwvIntercept', type=
'ArrayD', doc=
'ln(pwv) intercept parameter vector',
1595 size=pars[
'PARLNPWVINTERCEPT'].size)
1596 parSchema.addField(
'parLnPwvSlope', type=
'ArrayD', doc=
'ln(pwv) slope parameter vector',
1597 size=pars[
'PARLNPWVSLOPE'].size)
1598 parSchema.addField(
'parLnPwvQuadratic', type=
'ArrayD', doc=
'ln(pwv) quadratic parameter vector',
1599 size=pars[
'PARLNPWVQUADRATIC'].size)
1600 parSchema.addField(
'parQeSysIntercept', type=
'ArrayD', doc=
'Mirror gray intercept parameter vector',
1601 size=pars[
'PARQESYSINTERCEPT'].size)
1602 parSchema.addField(
'compQeSysSlope', type=
'ArrayD', doc=
'Mirror gray slope parameter vector',
1603 size=pars[0][
'COMPQESYSSLOPE'].size)
1604 parSchema.addField(
'parFilterOffset', type=
'ArrayD', doc=
'Filter offset parameter vector',
1605 size=pars[
'PARFILTEROFFSET'].size)
1606 parSchema.addField(
'parFilterOffsetFitFlag', type=
'ArrayI', doc=
'Filter offset parameter fit flag',
1607 size=pars[
'PARFILTEROFFSETFITFLAG'].size)
1608 parSchema.addField(
'parRetrievedLnPwvScale', type=np.float64,
1609 doc=
'Global scale for retrieved ln(pwv)')
1610 parSchema.addField(
'parRetrievedLnPwvOffset', type=np.float64,
1611 doc=
'Global offset for retrieved ln(pwv)')
1612 parSchema.addField(
'parRetrievedLnPwvNightlyOffset', type=
'ArrayD',
1613 doc=
'Nightly offset for retrieved ln(pwv)',
1614 size=pars[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1615 parSchema.addField(
'compAbsThroughput', type=
'ArrayD',
1616 doc=
'Absolute throughput (relative to transmission curves)',
1617 size=pars[
'COMPABSTHROUGHPUT'].size)
1618 parSchema.addField(
'compRefOffset', type=
'ArrayD',
1619 doc=
'Offset between reference stars and calibrated stars',
1620 size=pars[
'COMPREFOFFSET'].size)
1621 parSchema.addField(
'compRefSigma', type=
'ArrayD',
1622 doc=
'Width of reference star/calibrated star distribution',
1623 size=pars[
'COMPREFSIGMA'].size)
1624 parSchema.addField(
'compMirrorChromaticity', type=
'ArrayD',
1625 doc=
'Computed mirror chromaticity terms',
1626 size=pars[
'COMPMIRRORCHROMATICITY'].size)
1627 parSchema.addField(
'mirrorChromaticityPivot', type=
'ArrayD',
1628 doc=
'Mirror chromaticity pivot mjd',
1629 size=pars[
'MIRRORCHROMATICITYPIVOT'].size)
1630 parSchema.addField(
'compMedianSedSlope', type=
'ArrayD',
1631 doc=
'Computed median SED slope (per band)',
1632 size=pars[
'COMPMEDIANSEDSLOPE'].size)
1633 parSchema.addField(
'compAperCorrPivot', type=
'ArrayD', doc=
'Aperture correction pivot',
1634 size=pars[
'COMPAPERCORRPIVOT'].size)
1635 parSchema.addField(
'compAperCorrSlope', type=
'ArrayD', doc=
'Aperture correction slope',
1636 size=pars[
'COMPAPERCORRSLOPE'].size)
1637 parSchema.addField(
'compAperCorrSlopeErr', type=
'ArrayD', doc=
'Aperture correction slope error',
1638 size=pars[
'COMPAPERCORRSLOPEERR'].size)
1639 parSchema.addField(
'compAperCorrRange', type=
'ArrayD', doc=
'Aperture correction range',
1640 size=pars[
'COMPAPERCORRRANGE'].size)
1641 parSchema.addField(
'compModelErrExptimePivot', type=
'ArrayD', doc=
'Model error exptime pivot',
1642 size=pars[
'COMPMODELERREXPTIMEPIVOT'].size)
1643 parSchema.addField(
'compModelErrFwhmPivot', type=
'ArrayD', doc=
'Model error fwhm pivot',
1644 size=pars[
'COMPMODELERRFWHMPIVOT'].size)
1645 parSchema.addField(
'compModelErrSkyPivot', type=
'ArrayD', doc=
'Model error sky pivot',
1646 size=pars[
'COMPMODELERRSKYPIVOT'].size)
1647 parSchema.addField(
'compModelErrPars', type=
'ArrayD', doc=
'Model error parameters',
1648 size=pars[
'COMPMODELERRPARS'].size)
1649 parSchema.addField(
'compExpGray', type=
'ArrayD', doc=
'Computed exposure gray',
1650 size=pars[
'COMPEXPGRAY'].size)
1651 parSchema.addField(
'compVarGray', type=
'ArrayD', doc=
'Computed exposure variance',
1652 size=pars[
'COMPVARGRAY'].size)
1653 parSchema.addField(
'compExpDeltaMagBkg', type=
'ArrayD',
1654 doc=
'Computed exposure offset due to background',
1655 size=pars[
'COMPEXPDELTAMAGBKG'].size)
1656 parSchema.addField(
'compNGoodStarPerExp', type=
'ArrayI',
1657 doc=
'Computed number of good stars per exposure',
1658 size=pars[
'COMPNGOODSTARPEREXP'].size)
1659 parSchema.addField(
'compSigFgcm', type=
'ArrayD', doc=
'Computed sigma_fgcm (intrinsic repeatability)',
1660 size=pars[
'COMPSIGFGCM'].size)
1661 parSchema.addField(
'compSigmaCal', type=
'ArrayD', doc=
'Computed sigma_cal (systematic error floor)',
1662 size=pars[
'COMPSIGMACAL'].size)
1663 parSchema.addField(
'compRetrievedLnPwv', type=
'ArrayD', doc=
'Retrieved ln(pwv) (smoothed)',
1664 size=pars[
'COMPRETRIEVEDLNPWV'].size)
1665 parSchema.addField(
'compRetrievedLnPwvRaw', type=
'ArrayD', doc=
'Retrieved ln(pwv) (raw)',
1666 size=pars[
'COMPRETRIEVEDLNPWVRAW'].size)
1667 parSchema.addField(
'compRetrievedLnPwvFlag', type=
'ArrayI', doc=
'Retrieved ln(pwv) Flag',
1668 size=pars[
'COMPRETRIEVEDLNPWVFLAG'].size)
1669 parSchema.addField(
'compRetrievedTauNight', type=
'ArrayD', doc=
'Retrieved tau (per night)',
1670 size=pars[
'COMPRETRIEVEDTAUNIGHT'].size)
1672 parSchema.addField(
'superstarSize', type=
'ArrayI', doc=
'Superstar matrix size',
1674 parSchema.addField(
'superstar', type=
'ArrayD', doc=
'Superstar matrix (flattened)',
1675 size=parSuperStarFlat.size)
1679 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1680 lutFilterNameString, fitBandString):
1682 Make the FGCM parameter catalog for persistence
1686 parSchema: `lsst.afw.table.Schema`
1687 Parameter catalog schema
1688 pars: `numpy.ndarray`
1689 FGCM parameters to put into parCat
1690 parSuperStarFlat: `numpy.array`
1691 FGCM superstar flat array to put into parCat
1692 lutFilterNameString: `str`
1693 Combined string of all the lutFilterNames
1694 fitBandString: `str`
1695 Combined string of all the fitBands
1699 parCat: `afwTable.BasicCatalog`
1700 Atmosphere and instrumental model parameter catalog for persistence
1703 parCat = afwTable.BaseCatalog(parSchema)
1708 rec = parCat.addNew()
1711 rec[
'nCcd'] = parInfo[
'NCCD']
1712 rec[
'lutFilterNames'] = lutFilterNameString
1713 rec[
'fitBands'] = fitBandString
1715 rec[
'hasExternalPwv'] = 0
1716 rec[
'hasExternalTau'] = 0
1720 scalarNames = [
'parRetrievedLnPwvScale',
'parRetrievedLnPwvOffset']
1722 arrNames = [
'parAlpha',
'parO3',
'parLnTauIntercept',
'parLnTauSlope',
1723 'parLnPwvIntercept',
'parLnPwvSlope',
'parLnPwvQuadratic',
1724 'parQeSysIntercept',
'compQeSysSlope',
1725 'parRetrievedLnPwvNightlyOffset',
'compAperCorrPivot',
1726 'parFilterOffset',
'parFilterOffsetFitFlag',
1727 'compAbsThroughput',
'compRefOffset',
'compRefSigma',
1728 'compMirrorChromaticity',
'mirrorChromaticityPivot',
1729 'compAperCorrSlope',
'compAperCorrSlopeErr',
'compAperCorrRange',
1730 'compModelErrExptimePivot',
'compModelErrFwhmPivot',
1731 'compModelErrSkyPivot',
'compModelErrPars',
1732 'compExpGray',
'compVarGray',
'compNGoodStarPerExp',
'compSigFgcm',
1733 'compSigmaCal',
'compExpDeltaMagBkg',
'compMedianSedSlope',
1734 'compRetrievedLnPwv',
'compRetrievedLnPwvRaw',
'compRetrievedLnPwvFlag',
1735 'compRetrievedTauNight']
1737 for scalarName
in scalarNames:
1738 rec[scalarName] = pars[scalarName.upper()]
1740 for arrName
in arrNames:
1741 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1744 rec[
'superstarSize'][:] = parSuperStarFlat.shape
1745 rec[
'superstar'][:] = parSuperStarFlat.flatten()
1749 def _makeFlagStarSchema(self):
1751 Make the flagged-stars schema
1755 flagStarSchema: `lsst.afw.table.Schema`
1758 flagStarSchema = afwTable.Schema()
1760 flagStarSchema.addField(
'objId', type=np.int32, doc=
'FGCM object id')
1761 flagStarSchema.addField(
'objFlag', type=np.int32, doc=
'FGCM object flag')
1763 return flagStarSchema
1765 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1767 Make the flagged star catalog for persistence
1771 flagStarSchema: `lsst.afw.table.Schema`
1773 flagStarStruct: `numpy.ndarray`
1774 Flagged star structure from fgcm
1778 flagStarCat: `lsst.afw.table.BaseCatalog`
1779 Flagged star catalog for persistence
1782 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1783 flagStarCat.resize(flagStarStruct.size)
1785 flagStarCat[
'objId'][:] = flagStarStruct[
'OBJID']
1786 flagStarCat[
'objFlag'][:] = flagStarStruct[
'OBJFLAG']
def extractReferenceMags(refStars, bands, filterMap)
def makeStdSchema(nBands)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
def makeAtmCat(atmSchema, atmStruct)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def computeCcdOffsets(camera, defaultOrientation)
def translateVisitCatalog(visitCat)