23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table)
and fgcmBuildStars (to extract visits
and star observations
30The fgcmFitCycle
is meant to be run multiple times,
and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters
for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook
for details.
40import lsst.pex.config as pexConfig
41import lsst.pipe.base as pipeBase
42from lsst.pipe.base import connectionTypes
43import lsst.afw.table as afwTable
45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
46from .utilities import extractReferenceMags
47from .utilities import makeZptSchema, makeZptCat
48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
49from .sedterms import SedboundarytermDict, SedtermDict
50from .utilities import lookupStaticCalibrations
51from .focalPlaneProjector import FocalPlaneProjector
55__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask']
57MULTIPLE_CYCLES_MAX = 10
60class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
61 dimensions=("instrument",),
62 defaultTemplates={
"previousCycleNumber":
"-1",
64 camera = connectionTypes.PrerequisiteInput(
65 doc=
"Camera instrument",
67 storageClass=
"Camera",
68 dimensions=(
"instrument",),
69 lookupFunction=lookupStaticCalibrations,
73 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
74 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
75 "chromatic corrections."),
76 name=
"fgcmLookUpTable",
77 storageClass=
"Catalog",
78 dimensions=(
"instrument",),
82 fgcmVisitCatalog = connectionTypes.Input(
83 doc=
"Catalog of visit information for fgcm",
84 name=
"fgcmVisitCatalog",
85 storageClass=
"Catalog",
86 dimensions=(
"instrument",),
90 fgcmStarObservations = connectionTypes.Input(
91 doc=
"Catalog of star observations for fgcm",
92 name=
"fgcmStarObservations",
93 storageClass=
"Catalog",
94 dimensions=(
"instrument",),
98 fgcmStarIds = connectionTypes.Input(
99 doc=
"Catalog of fgcm calibration star IDs",
101 storageClass=
"Catalog",
102 dimensions=(
"instrument",),
106 fgcmStarIndices = connectionTypes.Input(
107 doc=
"Catalog of fgcm calibration star indices",
108 name=
"fgcmStarIndices",
109 storageClass=
"Catalog",
110 dimensions=(
"instrument",),
114 fgcmReferenceStars = connectionTypes.Input(
115 doc=
"Catalog of fgcm-matched reference stars",
116 name=
"fgcmReferenceStars",
117 storageClass=
"Catalog",
118 dimensions=(
"instrument",),
122 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
123 doc=
"Catalog of flagged stars for fgcm calibration from previous fit cycle",
124 name=
"fgcmFlaggedStars{previousCycleNumber}",
125 storageClass=
"Catalog",
126 dimensions=(
"instrument",),
130 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
131 doc=
"Catalog of fgcm fit parameters from previous fit cycle",
132 name=
"fgcmFitParameters{previousCycleNumber}",
133 storageClass=
"Catalog",
134 dimensions=(
"instrument",),
138 fgcmFitParameters = connectionTypes.Output(
139 doc=
"Catalog of fgcm fit parameters from current fit cycle",
140 name=
"fgcmFitParameters{cycleNumber}",
141 storageClass=
"Catalog",
142 dimensions=(
"instrument",),
145 fgcmFlaggedStars = connectionTypes.Output(
146 doc=
"Catalog of flagged stars for fgcm calibration from current fit cycle",
147 name=
"fgcmFlaggedStars{cycleNumber}",
148 storageClass=
"Catalog",
149 dimensions=(
"instrument",),
152 fgcmZeropoints = connectionTypes.Output(
153 doc=
"Catalog of fgcm zeropoint data from current fit cycle",
154 name=
"fgcmZeropoints{cycleNumber}",
155 storageClass=
"Catalog",
156 dimensions=(
"instrument",),
159 fgcmAtmosphereParameters = connectionTypes.Output(
160 doc=
"Catalog of atmospheric fit parameters from current fit cycle",
161 name=
"fgcmAtmosphereParameters{cycleNumber}",
162 storageClass=
"Catalog",
163 dimensions=(
"instrument",),
166 fgcmStandardStars = connectionTypes.Output(
167 doc=
"Catalog of standard star magnitudes from current fit cycle",
168 name=
"fgcmStandardStars{cycleNumber}",
169 storageClass=
"SimpleCatalog",
170 dimensions=(
"instrument",),
176 for cycle
in range(MULTIPLE_CYCLES_MAX):
177 vars()[f
"fgcmFitParameters{cycle}"] = connectionTypes.Output(
178 doc=f
"Catalog of fgcm fit parameters from fit cycle {cycle}",
179 name=f
"fgcmFitParameters{cycle}",
180 storageClass=
"Catalog",
181 dimensions=(
"instrument",),
183 vars()[f
"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
184 doc=f
"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
185 name=f
"fgcmFlaggedStars{cycle}",
186 storageClass=
"Catalog",
187 dimensions=(
"instrument",),
189 vars()[f
"fgcmZeropoints{cycle}"] = connectionTypes.Output(
190 doc=f
"Catalog of fgcm zeropoint data from fit cycle {cycle}",
191 name=f
"fgcmZeropoints{cycle}",
192 storageClass=
"Catalog",
193 dimensions=(
"instrument",),
195 vars()[f
"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
196 doc=f
"Catalog of atmospheric fit parameters from fit cycle {cycle}",
197 name=f
"fgcmAtmosphereParameters{cycle}",
198 storageClass=
"Catalog",
199 dimensions=(
"instrument",),
201 vars()[f
"fgcmStandardStars{cycle}"] = connectionTypes.Output(
202 doc=f
"Catalog of standard star magnitudes from fit cycle {cycle}",
203 name=f
"fgcmStandardStars{cycle}",
204 storageClass=
"SimpleCatalog",
205 dimensions=(
"instrument",),
208 def __init__(self, *, config=None):
209 super().__init__(config=config)
211 if not config.doReferenceCalibration:
212 self.inputs.remove(
"fgcmReferenceStars")
214 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
215 raise ValueError(
"cycleNumber must be of integer format")
216 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
217 raise ValueError(
"previousCycleNumber must be of integer format")
218 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
219 raise ValueError(
"previousCycleNumber must be 1 less than cycleNumber")
221 if int(config.connections.cycleNumber) == 0:
222 self.prerequisiteInputs.remove(
"fgcmFlaggedStarsInput")
223 self.prerequisiteInputs.remove(
"fgcmFitParametersInput")
225 if not self.config.doMultipleCycles:
227 if not self.config.isFinalCycle
and not self.config.outputStandardsBeforeFinalCycle:
228 self.outputs.remove(
"fgcmStandardStars")
230 if not self.config.isFinalCycle
and not self.config.outputZeropointsBeforeFinalCycle:
231 self.outputs.remove(
"fgcmZeropoints")
232 self.outputs.remove(
"fgcmAtmosphereParameters")
235 for cycle
in range(0, MULTIPLE_CYCLES_MAX):
236 self.outputs.remove(f
"fgcmFitParameters{cycle}")
237 self.outputs.remove(f
"fgcmFlaggedStars{cycle}")
238 self.outputs.remove(f
"fgcmZeropoints{cycle}")
239 self.outputs.remove(f
"fgcmAtmosphereParameters{cycle}")
240 self.outputs.remove(f
"fgcmStandardStars{cycle}")
245 self.outputs.remove(
"fgcmFitParameters")
246 self.outputs.remove(
"fgcmFlaggedStars")
247 self.outputs.remove(
"fgcmZeropoints")
248 self.outputs.remove(
"fgcmAtmosphereParameters")
249 self.outputs.remove(
"fgcmStandardStars")
252 for cycle
in range(self.config.multipleCyclesFinalCycleNumber + 1,
253 MULTIPLE_CYCLES_MAX):
254 self.outputs.remove(f
"fgcmFitParameters{cycle}")
255 self.outputs.remove(f
"fgcmFlaggedStars{cycle}")
256 self.outputs.remove(f
"fgcmZeropoints{cycle}")
257 self.outputs.remove(f
"fgcmAtmosphereParameters{cycle}")
258 self.outputs.remove(f
"fgcmStandardStars{cycle}")
261 for cycle
in range(self.config.multipleCyclesFinalCycleNumber):
262 if not self.config.outputZeropointsBeforeFinalCycle:
263 self.outputs.remove(f
"fgcmZeropoints{cycle}")
264 self.outputs.remove(f
"fgcmAtmosphereParameters{cycle}")
265 if not self.config.outputStandardsBeforeFinalCycle:
266 self.outputs.remove(f
"fgcmStandardStars{cycle}")
269class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
270 pipelineConnections=FgcmFitCycleConnections):
271 """Config for FgcmFitCycle"""
273 doMultipleCycles = pexConfig.Field(
274 doc=
"Run multiple fit cycles in one task",
278 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
279 doc=(
"Final cycle number in multiple cycle mode. The initial cycle "
280 "is 0, with limited parameters fit. The next cycle is 1 with "
281 "full parameter fit. The final cycle is a clean-up with no "
282 "parameters fit. There will be a total of "
283 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
284 "cycle number cannot be less than 2."),
288 max=MULTIPLE_CYCLES_MAX,
291 bands = pexConfig.ListField(
292 doc=
"Bands to run calibration",
296 fitBands = pexConfig.ListField(
297 doc=(
"Bands to use in atmospheric fit. The bands not listed here will have "
298 "the atmosphere constrained from the 'fitBands' on the same night. "
299 "Must be a subset of `config.bands`"),
303 requiredBands = pexConfig.ListField(
304 doc=(
"Bands that are required for a star to be considered a calibration star. "
305 "Must be a subset of `config.bands`"),
309 physicalFilterMap = pexConfig.DictField(
310 doc=
"Mapping from 'physicalFilter' to band.",
315 doReferenceCalibration = pexConfig.Field(
316 doc=
"Use reference catalog as additional constraint on calibration",
320 refStarSnMin = pexConfig.Field(
321 doc=
"Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
325 refStarOutlierNSig = pexConfig.Field(
326 doc=(
"Number of sigma compared to average mag for reference star to be considered an outlier. "
327 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
331 applyRefStarColorCuts = pexConfig.Field(
332 doc=
"Apply color cuts to reference stars?",
336 nCore = pexConfig.Field(
337 doc=
"Number of cores to use",
341 nStarPerRun = pexConfig.Field(
342 doc=
"Number of stars to run in each chunk",
346 nExpPerRun = pexConfig.Field(
347 doc=
"Number of exposures to run in each chunk",
351 reserveFraction = pexConfig.Field(
352 doc=
"Fraction of stars to reserve for testing",
356 freezeStdAtmosphere = pexConfig.Field(
357 doc=
"Freeze atmosphere parameters to standard (for testing)",
361 precomputeSuperStarInitialCycle = pexConfig.Field(
362 doc=
"Precompute superstar flat for initial cycle",
366 superStarSubCcdDict = pexConfig.DictField(
367 doc=(
"Per-band specification on whether to compute superstar flat on sub-ccd scale. "
368 "Must have one entry per band."),
373 superStarSubCcdChebyshevOrder = pexConfig.Field(
374 doc=(
"Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
375 "Global default is first-order polynomials, and should be overridden "
376 "on a camera-by-camera basis depending on the ISR."),
380 superStarSubCcdTriangular = pexConfig.Field(
381 doc=(
"Should the sub-ccd superstar chebyshev matrix be triangular to "
382 "suppress high-order cross terms?"),
386 superStarSigmaClip = pexConfig.Field(
387 doc=
"Number of sigma to clip outliers when selecting for superstar flats",
391 focalPlaneSigmaClip = pexConfig.Field(
392 doc=
"Number of sigma to clip outliers per focal-plane.",
396 ccdGraySubCcdDict = pexConfig.DictField(
397 doc=(
"Per-band specification on whether to compute achromatic per-ccd residual "
398 "('ccd gray') on a sub-ccd scale."),
403 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
404 doc=
"Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
408 ccdGraySubCcdTriangular = pexConfig.Field(
409 doc=(
"Should the sub-ccd gray chebyshev matrix be triangular to "
410 "suppress high-order cross terms?"),
414 ccdGrayFocalPlaneDict = pexConfig.DictField(
415 doc=(
"Per-band specification on whether to compute focal-plane residual "
416 "('ccd gray') corrections."),
421 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
422 doc=(
"Minimum number of 'good' CCDs required to perform focal-plane "
423 "gray corrections. If there are fewer good CCDs then the gray "
424 "correction is computed per-ccd."),
428 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
429 doc=
"Order of the 2D chebyshev polynomials for focal plane fit.",
433 cycleNumber = pexConfig.Field(
434 doc=(
"FGCM fit cycle number. This is automatically incremented after each run "
435 "and stage of outlier rejection. See cookbook for details."),
439 isFinalCycle = pexConfig.Field(
440 doc=(
"Is this the final cycle of the fitting? Will automatically compute final "
441 "selection of stars and photometric exposures, and will output zeropoints "
442 "and standard stars for use in fgcmOutputProducts"),
446 maxIterBeforeFinalCycle = pexConfig.Field(
447 doc=(
"Maximum fit iterations, prior to final cycle. The number of iterations "
448 "will always be 0 in the final cycle for cleanup and final selection."),
452 deltaMagBkgOffsetPercentile = pexConfig.Field(
453 doc=(
"Percentile brightest stars on a visit/ccd to use to compute net "
454 "offset from local background subtraction."),
458 deltaMagBkgPerCcd = pexConfig.Field(
459 doc=(
"Compute net offset from local background subtraction per-ccd? "
460 "Otherwise, use computation per visit."),
464 utBoundary = pexConfig.Field(
465 doc=
"Boundary (in UTC) from day-to-day",
469 washMjds = pexConfig.ListField(
470 doc=
"Mirror wash MJDs",
474 epochMjds = pexConfig.ListField(
475 doc=
"Epoch boundaries in MJD",
479 minObsPerBand = pexConfig.Field(
480 doc=
"Minimum good observations per band",
486 latitude = pexConfig.Field(
487 doc=
"Observatory latitude",
491 defaultCameraOrientation = pexConfig.Field(
492 doc=
"Default camera orientation for QA plots.",
496 brightObsGrayMax = pexConfig.Field(
497 doc=
"Maximum gray extinction to be considered bright observation",
501 minStarPerCcd = pexConfig.Field(
502 doc=(
"Minimum number of good stars per CCD to be used in calibration fit. "
503 "CCDs with fewer stars will have their calibration estimated from other "
504 "CCDs in the same visit, with zeropoint error increased accordingly."),
508 minCcdPerExp = pexConfig.Field(
509 doc=(
"Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
510 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
514 maxCcdGrayErr = pexConfig.Field(
515 doc=
"Maximum error on CCD gray offset to be considered photometric",
519 minStarPerExp = pexConfig.Field(
520 doc=(
"Minimum number of good stars per exposure/visit to be used in calibration fit. "
521 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
525 minExpPerNight = pexConfig.Field(
526 doc=
"Minimum number of good exposures/visits to consider a partly photometric night",
530 expGrayInitialCut = pexConfig.Field(
531 doc=(
"Maximum exposure/visit gray value for initial selection of possible photometric "
536 expGrayPhotometricCutDict = pexConfig.DictField(
537 doc=(
"Per-band specification on maximum (negative) achromatic exposure residual "
538 "('gray term') for a visit to be considered photometric. Must have one "
539 "entry per band. Broad-band filters should be -0.05."),
544 expGrayHighCutDict = pexConfig.DictField(
545 doc=(
"Per-band specification on maximum (positive) achromatic exposure residual "
546 "('gray term') for a visit to be considered photometric. Must have one "
547 "entry per band. Broad-band filters should be 0.2."),
552 expGrayRecoverCut = pexConfig.Field(
553 doc=(
"Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
554 "Visits with more gray extinction will only get CCD zeropoints if there are "
555 "sufficient star observations (minStarPerCcd) on that CCD."),
559 expVarGrayPhotometricCutDict = pexConfig.DictField(
560 doc=(
"Per-band specification on maximum exposure variance to be considered possibly "
561 "photometric. Must have one entry per band. Broad-band filters should be "
567 expGrayErrRecoverCut = pexConfig.Field(
568 doc=(
"Maximum exposure gray error to be able to recover bad ccds via interpolation. "
569 "Visits with more gray variance will only get CCD zeropoints if there are "
570 "sufficient star observations (minStarPerCcd) on that CCD."),
574 aperCorrFitNBins = pexConfig.Field(
575 doc=(
"Number of aperture bins used in aperture correction fit. When set to 0"
576 "no fit will be performed, and the config.aperCorrInputSlopes will be "
577 "used if available."),
581 aperCorrInputSlopeDict = pexConfig.DictField(
582 doc=(
"Per-band specification of aperture correction input slope parameters. These "
583 "are used on the first fit iteration, and aperture correction parameters will "
584 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
585 "to set this when there is insufficient data to fit the parameters (e.g. "
591 sedboundaryterms = pexConfig.ConfigField(
592 doc=
"Mapping from bands to SED boundary term names used is sedterms.",
593 dtype=SedboundarytermDict,
595 sedterms = pexConfig.ConfigField(
596 doc=
"Mapping from terms to bands for fgcm linear SED approximations.",
599 sigFgcmMaxErr = pexConfig.Field(
600 doc=
"Maximum mag error for fitting sigma_FGCM",
604 sigFgcmMaxEGrayDict = pexConfig.DictField(
605 doc=(
"Per-band specification for maximum (absolute) achromatic residual (gray value) "
606 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
612 ccdGrayMaxStarErr = pexConfig.Field(
613 doc=(
"Maximum error on a star observation to use in ccd gray (achromatic residual) "
618 approxThroughputDict = pexConfig.DictField(
619 doc=(
"Per-band specification of the approximate overall throughput at the start of "
620 "calibration observations. Must have one entry per band. Typically should "
626 sigmaCalRange = pexConfig.ListField(
627 doc=
"Allowed range for systematic error floor estimation",
629 default=(0.001, 0.003),
631 sigmaCalFitPercentile = pexConfig.ListField(
632 doc=
"Magnitude percentile range to fit systematic error floor",
634 default=(0.05, 0.15),
636 sigmaCalPlotPercentile = pexConfig.ListField(
637 doc=
"Magnitude percentile range to plot systematic error floor",
639 default=(0.05, 0.95),
641 sigma0Phot = pexConfig.Field(
642 doc=
"Systematic error floor for all zeropoints",
646 mapLongitudeRef = pexConfig.Field(
647 doc=
"Reference longitude for plotting maps",
651 mapNSide = pexConfig.Field(
652 doc=
"Healpix nside for plotting maps",
656 outfileBase = pexConfig.Field(
657 doc=
"Filename start for plot output files",
661 starColorCuts = pexConfig.ListField(
662 doc=
"Encoded star-color cuts (to be cleaned up)",
664 default=(
"NO_DATA",),
666 colorSplitBands = pexConfig.ListField(
667 doc=
"Band names to use to split stars by color. Must have 2 entries.",
672 modelMagErrors = pexConfig.Field(
673 doc=
"Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
677 useQuadraticPwv = pexConfig.Field(
678 doc=
"Model PWV with a quadratic term for variation through the night?",
682 instrumentParsPerBand = pexConfig.Field(
683 doc=(
"Model instrumental parameters per band? "
684 "Otherwise, instrumental parameters (QE changes with time) are "
685 "shared among all bands."),
689 instrumentSlopeMinDeltaT = pexConfig.Field(
690 doc=(
"Minimum time change (in days) between observations to use in constraining "
691 "instrument slope."),
695 fitMirrorChromaticity = pexConfig.Field(
696 doc=
"Fit (intraband) mirror chromatic term?",
700 coatingMjds = pexConfig.ListField(
701 doc=
"Mirror coating dates in MJD",
705 outputStandardsBeforeFinalCycle = pexConfig.Field(
706 doc=
"Output standard stars prior to final cycle? Used in debugging.",
710 outputZeropointsBeforeFinalCycle = pexConfig.Field(
711 doc=
"Output standard stars prior to final cycle? Used in debugging.",
715 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
716 doc=(
"Per-band specification on whether to use star repeatability (instead of exposures) "
717 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
722 autoPhotometricCutNSig = pexConfig.Field(
723 doc=(
"Number of sigma for automatic computation of (low) photometric cut. "
724 "Cut is based on exposure gray width (per band), unless "
725 "useRepeatabilityForExpGrayCuts is set, in which case the star "
726 "repeatability is used (also per band)."),
730 autoHighCutNSig = pexConfig.Field(
731 doc=(
"Number of sigma for automatic computation of (high) outlier cut. "
732 "Cut is based on exposure gray width (per band), unless "
733 "useRepeatabilityForExpGrayCuts is set, in which case the star "
734 "repeatability is used (also per band)."),
738 quietMode = pexConfig.Field(
739 doc=
"Be less verbose with logging.",
743 doPlots = pexConfig.Field(
744 doc=
"Make fgcm QA plots.",
748 randomSeed = pexConfig.Field(
749 doc=
"Random seed for fgcm for consistency in tests.",
754 deltaAperFitMinNgoodObs = pexConfig.Field(
755 doc=
"Minimum number of good observations to use mean delta-aper values in fits.",
759 deltaAperFitPerCcdNx = pexConfig.Field(
760 doc=(
"Number of x bins per ccd when computing delta-aper background offsets. "
761 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
765 deltaAperFitPerCcdNy = pexConfig.Field(
766 doc=(
"Number of y bins per ccd when computing delta-aper background offsets. "
767 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
771 deltaAperFitSpatialNside = pexConfig.Field(
772 doc=
"Healpix nside to compute spatial delta-aper background offset maps.",
776 deltaAperInnerRadiusArcsec = pexConfig.Field(
777 doc=(
"Inner radius used to compute deltaMagAper (arcseconds). "
778 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if "
779 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
780 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
784 deltaAperOuterRadiusArcsec = pexConfig.Field(
785 doc=(
"Outer radius used to compute deltaMagAper (arcseconds). "
786 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if "
787 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
788 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
792 doComputeDeltaAperPerVisit = pexConfig.Field(
793 doc=(
"Do the computation of delta-aper background offsets per visit? "
794 "Note: this option can be very slow when there are many visits."),
798 doComputeDeltaAperPerStar = pexConfig.Field(
799 doc=
"Do the computation of delta-aper mean values per star?",
803 doComputeDeltaAperMap = pexConfig.Field(
804 doc=(
"Do the computation of delta-aper spatial maps? "
805 "This is only used if ``doComputeDeltaAperPerStar`` is True,"),
809 doComputeDeltaAperPerCcd = pexConfig.Field(
810 doc=
"Do the computation of per-ccd delta-aper background offsets?",
818 if self.connections.previousCycleNumber !=
str(self.cycleNumber - 1):
819 msg =
"cycleNumber in template must be connections.previousCycleNumber + 1"
820 raise RuntimeError(msg)
821 if self.connections.cycleNumber !=
str(self.cycleNumber):
822 msg =
"cycleNumber in template must be equal to connections.cycleNumber"
823 raise RuntimeError(msg)
825 for band
in self.fitBands:
826 if band
not in self.bands:
827 msg =
'fitBand %s not in bands' % (band)
828 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
829 for band
in self.requiredBands:
830 if band
not in self.bands:
831 msg =
'requiredBand %s not in bands' % (band)
832 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
833 for band
in self.colorSplitBands:
834 if band
not in self.bands:
835 msg =
'colorSplitBand %s not in bands' % (band)
836 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
837 for band
in self.bands:
838 if band
not in self.superStarSubCcdDict:
839 msg =
'band %s not in superStarSubCcdDict' % (band)
840 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
842 if band
not in self.ccdGraySubCcdDict:
843 msg =
'band %s not in ccdGraySubCcdDict' % (band)
844 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
846 if band
not in self.expGrayPhotometricCutDict:
847 msg =
'band %s not in expGrayPhotometricCutDict' % (band)
848 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
850 if band
not in self.expGrayHighCutDict:
851 msg =
'band %s not in expGrayHighCutDict' % (band)
852 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
854 if band
not in self.expVarGrayPhotometricCutDict:
855 msg =
'band %s not in expVarGrayPhotometricCutDict' % (band)
856 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
858 if band
not in self.sigFgcmMaxEGrayDict:
859 msg =
'band %s not in sigFgcmMaxEGrayDict' % (band)
860 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
862 if band
not in self.approxThroughputDict:
863 msg =
'band %s not in approxThroughputDict' % (band)
864 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
866 if band
not in self.useRepeatabilityForExpGrayCutsDict:
867 msg =
'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
868 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
871 if self.doComputeDeltaAperPerVisit
or self.doComputeDeltaAperMap \
872 or self.doComputeDeltaAperPerCcd:
873 if self.deltaAperInnerRadiusArcsec <= 0.0:
874 msg =
'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.'
875 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec,
877 if self.deltaAperOuterRadiusArcsec <= 0.0:
878 msg =
'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.'
879 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
881 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec:
882 msg = (
'deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if '
883 'deltaAper computations are turned on.')
884 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
888class FgcmFitCycleTask(pipeBase.PipelineTask):
890 Run Single fit cycle for FGCM
global calibration
893 ConfigClass = FgcmFitCycleConfig
894 _DefaultName = "fgcmFitCycle"
896 def __init__(self, initInputs=None, **kwargs):
897 super().__init__(**kwargs)
899 def runQuantum(self, butlerQC, inputRefs, outputRefs):
900 camera = butlerQC.get(inputRefs.camera)
904 handleDict[
'fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
905 handleDict[
'fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
906 handleDict[
'fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
907 handleDict[
'fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
908 handleDict[
'fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
909 if self.config.doReferenceCalibration:
910 handleDict[
'fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
911 if self.config.cycleNumber > 0:
912 handleDict[
'fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
913 handleDict[
'fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
915 fgcmDatasetDict =
None
916 if self.config.doMultipleCycles:
918 config = copy.copy(self.config)
919 config.update(cycleNumber=0)
920 for cycle
in range(self.config.multipleCyclesFinalCycleNumber + 1):
921 if cycle == self.config.multipleCyclesFinalCycleNumber:
922 config.update(isFinalCycle=
True)
925 handleDict[
'fgcmFlaggedStars'] = fgcmDatasetDict[
'fgcmFlaggedStars']
926 handleDict[
'fgcmFitParameters'] = fgcmDatasetDict[
'fgcmFitParameters']
928 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config)
929 butlerQC.put(fgcmDatasetDict[
'fgcmFitParameters'],
930 getattr(outputRefs, f
'fgcmFitParameters{cycle}'))
931 butlerQC.put(fgcmDatasetDict[
'fgcmFlaggedStars'],
932 getattr(outputRefs, f
'fgcmFlaggedStars{cycle}'))
933 if self.outputZeropoints:
934 butlerQC.put(fgcmDatasetDict[
'fgcmZeropoints'],
935 getattr(outputRefs, f
'fgcmZeropoints{cycle}'))
936 butlerQC.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'],
937 getattr(outputRefs, f
'fgcmAtmosphereParameters{cycle}'))
938 if self.outputStandards:
939 butlerQC.put(fgcmDatasetDict[
'fgcmStandardStars'],
940 getattr(outputRefs, f
'fgcmStandardStars{cycle}'))
943 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict)
945 butlerQC.put(fgcmDatasetDict[
'fgcmFitParameters'], outputRefs.fgcmFitParameters)
946 butlerQC.put(fgcmDatasetDict[
'fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
947 if self.outputZeropoints:
948 butlerQC.put(fgcmDatasetDict[
'fgcmZeropoints'], outputRefs.fgcmZeropoints)
949 butlerQC.put(fgcmDatasetDict[
'fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
950 if self.outputStandards:
951 butlerQC.put(fgcmDatasetDict[
'fgcmStandardStars'], outputRefs.fgcmStandardStars)
953 def _fgcmFitCycle(self, camera, handleDict, config=None):
959 camera : `lsst.afw.cameraGeom.Camera`
961 All handles are `lsst.daf.butler.DeferredDatasetHandle`
962 handle dictionary with keys:
964 ``
"fgcmLookUpTable"``
965 handle
for the FGCM look-up table.
966 ``
"fgcmVisitCatalog"``
967 handle
for visit summary catalog.
968 ``
"fgcmStarObservations"``
969 handle
for star observation catalog.
971 handle
for star id catalog.
972 ``
"fgcmStarIndices"``
973 handle
for star index catalog.
974 ``
"fgcmReferenceStars"``
975 handle
for matched reference star catalog.
976 ``
"fgcmFlaggedStars"``
977 handle
for flagged star catalog.
978 ``
"fgcmFitParameters"``
979 handle
for fit parameter catalog.
980 config : `lsst.pex.config.Config`, optional
981 Configuration to use to override self.config.
985 fgcmDatasetDict : `dict`
986 Dictionary of datasets to persist.
988 if config
is not None:
991 _config = self.config
994 self.maxIter = _config.maxIterBeforeFinalCycle
995 self.outputStandards = _config.outputStandardsBeforeFinalCycle
996 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
997 self.resetFitParameters =
True
999 if _config.isFinalCycle:
1004 self.outputStandards =
True
1005 self.outputZeropoints =
True
1006 self.resetFitParameters =
False
1008 lutCat = handleDict[
'fgcmLookUpTable'].get()
1010 dict(_config.physicalFilterMap))
1014 self.maxIter, self.resetFitParameters,
1015 self.outputZeropoints,
1016 lutIndexVals[0][
'FILTERNAMES'])
1019 visitCat = handleDict[
'fgcmVisitCatalog'].get()
1024 self.config.defaultCameraOrientation)
1026 noFitsDict = {
'lutIndex': lutIndexVals,
1028 'expInfo': fgcmExpInfo,
1029 'focalPlaneProjector': focalPlaneProjector}
1032 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
1033 noFitsDict=noFitsDict, noOutput=
True)
1036 if (fgcmFitCycle.initialCycle):
1038 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1042 if isinstance(handleDict[
'fgcmFitParameters'], afwTable.BaseCatalog):
1043 parCat = handleDict[
'fgcmFitParameters']
1045 parCat = handleDict[
'fgcmFitParameters'].get()
1046 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1048 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1055 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1057 starObs = handleDict[
'fgcmStarObservations'].get()
1058 starIds = handleDict[
'fgcmStarIds'].get()
1059 starIndices = handleDict[
'fgcmStarIndices'].get()
1062 if 'fgcmFlaggedStars' in handleDict:
1063 if isinstance(handleDict[
'fgcmFlaggedStars'], afwTable.BaseCatalog):
1064 flaggedStars = handleDict[
'fgcmFlaggedStars']
1066 flaggedStars = handleDict[
'fgcmFlaggedStars'].get()
1067 flagId = flaggedStars[
'objId'][:]
1068 flagFlag = flaggedStars[
'objFlag'][:]
1074 if _config.doReferenceCalibration:
1075 refStars = handleDict[
'fgcmReferenceStars'].get()
1079 _config.physicalFilterMap)
1080 refId = refStars[
'fgcm_id'][:]
1090 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'], starObs[
'visit'][starIndices[
'obsIndex']])
1102 conv = starObs[0][
'ra'].asDegrees() / float(starObs[0][
'ra'])
1104 fgcmStars.loadStars(fgcmPars,
1105 starObs[
'visit'][starIndices[
'obsIndex']],
1106 starObs[
'ccd'][starIndices[
'obsIndex']],
1107 starObs[
'ra'][starIndices[
'obsIndex']] * conv,
1108 starObs[
'dec'][starIndices[
'obsIndex']] * conv,
1109 starObs[
'instMag'][starIndices[
'obsIndex']],
1110 starObs[
'instMagErr'][starIndices[
'obsIndex']],
1111 fgcmExpInfo[
'FILTERNAME'][visitIndex],
1112 starIds[
'fgcm_id'][:],
1115 starIds[
'obsArrIndex'][:],
1117 obsX=starObs[
'x'][starIndices[
'obsIndex']],
1118 obsY=starObs[
'y'][starIndices[
'obsIndex']],
1119 obsDeltaMagBkg=starObs[
'deltaMagBkg'][starIndices[
'obsIndex']],
1120 obsDeltaAper=starObs[
'deltaMagAper'][starIndices[
'obsIndex']],
1121 psfCandidate=starObs[
'psf_candidate'][starIndices[
'obsIndex']],
1124 refMagErr=refMagErr,
1142 fgcmFitCycle.setLUT(fgcmLut)
1143 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1144 fgcmFitCycle.setPars(fgcmPars)
1147 fgcmFitCycle.finishSetup()
1156 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1161 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i])
for
1162 i, b
in enumerate(_config.bands)}
1163 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i])
for
1164 i, band
in enumerate(_config.bands)}
1166 outConfig = copy.copy(_config)
1167 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1168 precomputeSuperStarInitialCycle=
False,
1169 freezeStdAtmosphere=
False,
1170 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1171 expGrayHighCutDict=updatedHighCutDict)
1173 outConfig.connections.update(previousCycleNumber=
str(_config.cycleNumber),
1174 cycleNumber=
str(_config.cycleNumber + 1))
1176 configFileName =
'%s_cycle%02d_config.py' % (outConfig.outfileBase,
1177 outConfig.cycleNumber)
1178 outConfig.save(configFileName)
1180 if _config.isFinalCycle == 1:
1182 self.log.info(
"Everything is in place to run fgcmOutputProducts.py")
1184 self.log.info(
"Saved config for next cycle to %s" % (configFileName))
1185 self.log.info(
"Be sure to look at:")
1186 self.log.info(
" config.expGrayPhotometricCut")
1187 self.log.info(
" config.expGrayHighCut")
1188 self.log.info(
"If you are satisfied with the fit, please set:")
1189 self.log.info(
" config.isFinalCycle = True")
1191 fgcmFitCycle.freeSharedMemory()
1193 return fgcmDatasetDict, outConfig
1195 def _loadParameters(self, parCat):
1197 Load FGCM parameters from a previous fit cycle
1201 parCat : `lsst.afw.table.BaseCatalog`
1202 Parameter catalog
in afw table form.
1206 inParInfo: `numpy.ndarray`
1207 Numpy array parameter information formatted
for input to fgcm
1208 inParameters: `numpy.ndarray`
1209 Numpy array parameter values formatted
for input to fgcm
1210 inSuperStar: `numpy.array`
1211 Superstar flat formatted
for input to fgcm
1213 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(
','))
1214 parFitBands = np.array(parCat[0][
'fitBands'].split(
','))
1216 inParInfo = np.zeros(1, dtype=[(
'NCCD',
'i4'),
1217 (
'LUTFILTERNAMES', parLutFilterNames.dtype.str,
1218 (parLutFilterNames.size, )),
1219 (
'FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1220 (
'LNTAUUNIT',
'f8'),
1221 (
'LNTAUSLOPEUNIT',
'f8'),
1222 (
'ALPHAUNIT',
'f8'),
1223 (
'LNPWVUNIT',
'f8'),
1224 (
'LNPWVSLOPEUNIT',
'f8'),
1225 (
'LNPWVQUADRATICUNIT',
'f8'),
1226 (
'LNPWVGLOBALUNIT',
'f8'),
1228 (
'QESYSUNIT',
'f8'),
1229 (
'FILTEROFFSETUNIT',
'f8'),
1230 (
'HASEXTERNALPWV',
'i2'),
1231 (
'HASEXTERNALTAU',
'i2')])
1232 inParInfo[
'NCCD'] = parCat[
'nCcd']
1233 inParInfo[
'LUTFILTERNAMES'][:] = parLutFilterNames
1234 inParInfo[
'FITBANDS'][:] = parFitBands
1235 inParInfo[
'HASEXTERNALPWV'] = parCat[
'hasExternalPwv']
1236 inParInfo[
'HASEXTERNALTAU'] = parCat[
'hasExternalTau']
1238 inParams = np.zeros(1, dtype=[(
'PARALPHA',
'f8', (parCat[
'parAlpha'].size, )),
1239 (
'PARO3',
'f8', (parCat[
'parO3'].size, )),
1240 (
'PARLNTAUINTERCEPT',
'f8',
1241 (parCat[
'parLnTauIntercept'].size, )),
1242 (
'PARLNTAUSLOPE',
'f8',
1243 (parCat[
'parLnTauSlope'].size, )),
1244 (
'PARLNPWVINTERCEPT',
'f8',
1245 (parCat[
'parLnPwvIntercept'].size, )),
1246 (
'PARLNPWVSLOPE',
'f8',
1247 (parCat[
'parLnPwvSlope'].size, )),
1248 (
'PARLNPWVQUADRATIC',
'f8',
1249 (parCat[
'parLnPwvQuadratic'].size, )),
1250 (
'PARQESYSINTERCEPT',
'f8',
1251 (parCat[
'parQeSysIntercept'].size, )),
1252 (
'COMPQESYSSLOPE',
'f8',
1253 (parCat[
'compQeSysSlope'].size, )),
1254 (
'PARFILTEROFFSET',
'f8',
1255 (parCat[
'parFilterOffset'].size, )),
1256 (
'PARFILTEROFFSETFITFLAG',
'i2',
1257 (parCat[
'parFilterOffsetFitFlag'].size, )),
1258 (
'PARRETRIEVEDLNPWVSCALE',
'f8'),
1259 (
'PARRETRIEVEDLNPWVOFFSET',
'f8'),
1260 (
'PARRETRIEVEDLNPWVNIGHTLYOFFSET',
'f8',
1261 (parCat[
'parRetrievedLnPwvNightlyOffset'].size, )),
1262 (
'COMPABSTHROUGHPUT',
'f8',
1263 (parCat[
'compAbsThroughput'].size, )),
1264 (
'COMPREFOFFSET',
'f8',
1265 (parCat[
'compRefOffset'].size, )),
1266 (
'COMPREFSIGMA',
'f8',
1267 (parCat[
'compRefSigma'].size, )),
1268 (
'COMPMIRRORCHROMATICITY',
'f8',
1269 (parCat[
'compMirrorChromaticity'].size, )),
1270 (
'MIRRORCHROMATICITYPIVOT',
'f8',
1271 (parCat[
'mirrorChromaticityPivot'].size, )),
1272 (
'COMPMEDIANSEDSLOPE',
'f8',
1273 (parCat[
'compMedianSedSlope'].size, )),
1274 (
'COMPAPERCORRPIVOT',
'f8',
1275 (parCat[
'compAperCorrPivot'].size, )),
1276 (
'COMPAPERCORRSLOPE',
'f8',
1277 (parCat[
'compAperCorrSlope'].size, )),
1278 (
'COMPAPERCORRSLOPEERR',
'f8',
1279 (parCat[
'compAperCorrSlopeErr'].size, )),
1280 (
'COMPAPERCORRRANGE',
'f8',
1281 (parCat[
'compAperCorrRange'].size, )),
1282 (
'COMPMODELERREXPTIMEPIVOT',
'f8',
1283 (parCat[
'compModelErrExptimePivot'].size, )),
1284 (
'COMPMODELERRFWHMPIVOT',
'f8',
1285 (parCat[
'compModelErrFwhmPivot'].size, )),
1286 (
'COMPMODELERRSKYPIVOT',
'f8',
1287 (parCat[
'compModelErrSkyPivot'].size, )),
1288 (
'COMPMODELERRPARS',
'f8',
1289 (parCat[
'compModelErrPars'].size, )),
1290 (
'COMPEXPGRAY',
'f8',
1291 (parCat[
'compExpGray'].size, )),
1292 (
'COMPVARGRAY',
'f8',
1293 (parCat[
'compVarGray'].size, )),
1294 (
'COMPEXPDELTAMAGBKG',
'f8',
1295 (parCat[
'compExpDeltaMagBkg'].size, )),
1296 (
'COMPNGOODSTARPEREXP',
'i4',
1297 (parCat[
'compNGoodStarPerExp'].size, )),
1298 (
'COMPSIGFGCM',
'f8',
1299 (parCat[
'compSigFgcm'].size, )),
1300 (
'COMPSIGMACAL',
'f8',
1301 (parCat[
'compSigmaCal'].size, )),
1302 (
'COMPRETRIEVEDLNPWV',
'f8',
1303 (parCat[
'compRetrievedLnPwv'].size, )),
1304 (
'COMPRETRIEVEDLNPWVRAW',
'f8',
1305 (parCat[
'compRetrievedLnPwvRaw'].size, )),
1306 (
'COMPRETRIEVEDLNPWVFLAG',
'i2',
1307 (parCat[
'compRetrievedLnPwvFlag'].size, )),
1308 (
'COMPRETRIEVEDTAUNIGHT',
'f8',
1309 (parCat[
'compRetrievedTauNight'].size, )),
1310 (
'COMPEPSILON',
'f8',
1311 (parCat[
'compEpsilon'].size, )),
1312 (
'COMPMEDDELTAAPER',
'f8',
1313 (parCat[
'compMedDeltaAper'].size, )),
1314 (
'COMPGLOBALEPSILON',
'f4',
1315 (parCat[
'compGlobalEpsilon'].size, )),
1316 (
'COMPEPSILONMAP',
'f4',
1317 (parCat[
'compEpsilonMap'].size, )),
1318 (
'COMPEPSILONNSTARMAP',
'i4',
1319 (parCat[
'compEpsilonNStarMap'].size, )),
1320 (
'COMPEPSILONCCDMAP',
'f4',
1321 (parCat[
'compEpsilonCcdMap'].size, )),
1322 (
'COMPEPSILONCCDNSTARMAP',
'i4',
1323 (parCat[
'compEpsilonCcdNStarMap'].size, ))])
1325 inParams[
'PARALPHA'][:] = parCat[
'parAlpha'][0, :]
1326 inParams[
'PARO3'][:] = parCat[
'parO3'][0, :]
1327 inParams[
'PARLNTAUINTERCEPT'][:] = parCat[
'parLnTauIntercept'][0, :]
1328 inParams[
'PARLNTAUSLOPE'][:] = parCat[
'parLnTauSlope'][0, :]
1329 inParams[
'PARLNPWVINTERCEPT'][:] = parCat[
'parLnPwvIntercept'][0, :]
1330 inParams[
'PARLNPWVSLOPE'][:] = parCat[
'parLnPwvSlope'][0, :]
1331 inParams[
'PARLNPWVQUADRATIC'][:] = parCat[
'parLnPwvQuadratic'][0, :]
1332 inParams[
'PARQESYSINTERCEPT'][:] = parCat[
'parQeSysIntercept'][0, :]
1333 inParams[
'COMPQESYSSLOPE'][:] = parCat[
'compQeSysSlope'][0, :]
1334 inParams[
'PARFILTEROFFSET'][:] = parCat[
'parFilterOffset'][0, :]
1335 inParams[
'PARFILTEROFFSETFITFLAG'][:] = parCat[
'parFilterOffsetFitFlag'][0, :]
1336 inParams[
'PARRETRIEVEDLNPWVSCALE'] = parCat[
'parRetrievedLnPwvScale']
1337 inParams[
'PARRETRIEVEDLNPWVOFFSET'] = parCat[
'parRetrievedLnPwvOffset']
1338 inParams[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat[
'parRetrievedLnPwvNightlyOffset'][0, :]
1339 inParams[
'COMPABSTHROUGHPUT'][:] = parCat[
'compAbsThroughput'][0, :]
1340 inParams[
'COMPREFOFFSET'][:] = parCat[
'compRefOffset'][0, :]
1341 inParams[
'COMPREFSIGMA'][:] = parCat[
'compRefSigma'][0, :]
1342 inParams[
'COMPMIRRORCHROMATICITY'][:] = parCat[
'compMirrorChromaticity'][0, :]
1343 inParams[
'MIRRORCHROMATICITYPIVOT'][:] = parCat[
'mirrorChromaticityPivot'][0, :]
1344 inParams[
'COMPMEDIANSEDSLOPE'][:] = parCat[
'compMedianSedSlope'][0, :]
1345 inParams[
'COMPAPERCORRPIVOT'][:] = parCat[
'compAperCorrPivot'][0, :]
1346 inParams[
'COMPAPERCORRSLOPE'][:] = parCat[
'compAperCorrSlope'][0, :]
1347 inParams[
'COMPAPERCORRSLOPEERR'][:] = parCat[
'compAperCorrSlopeErr'][0, :]
1348 inParams[
'COMPAPERCORRRANGE'][:] = parCat[
'compAperCorrRange'][0, :]
1349 inParams[
'COMPMODELERREXPTIMEPIVOT'][:] = parCat[
'compModelErrExptimePivot'][0, :]
1350 inParams[
'COMPMODELERRFWHMPIVOT'][:] = parCat[
'compModelErrFwhmPivot'][0, :]
1351 inParams[
'COMPMODELERRSKYPIVOT'][:] = parCat[
'compModelErrSkyPivot'][0, :]
1352 inParams[
'COMPMODELERRPARS'][:] = parCat[
'compModelErrPars'][0, :]
1353 inParams[
'COMPEXPGRAY'][:] = parCat[
'compExpGray'][0, :]
1354 inParams[
'COMPVARGRAY'][:] = parCat[
'compVarGray'][0, :]
1355 inParams[
'COMPEXPDELTAMAGBKG'][:] = parCat[
'compExpDeltaMagBkg'][0, :]
1356 inParams[
'COMPNGOODSTARPEREXP'][:] = parCat[
'compNGoodStarPerExp'][0, :]
1357 inParams[
'COMPSIGFGCM'][:] = parCat[
'compSigFgcm'][0, :]
1358 inParams[
'COMPSIGMACAL'][:] = parCat[
'compSigmaCal'][0, :]
1359 inParams[
'COMPRETRIEVEDLNPWV'][:] = parCat[
'compRetrievedLnPwv'][0, :]
1360 inParams[
'COMPRETRIEVEDLNPWVRAW'][:] = parCat[
'compRetrievedLnPwvRaw'][0, :]
1361 inParams[
'COMPRETRIEVEDLNPWVFLAG'][:] = parCat[
'compRetrievedLnPwvFlag'][0, :]
1362 inParams[
'COMPRETRIEVEDTAUNIGHT'][:] = parCat[
'compRetrievedTauNight'][0, :]
1363 inParams[
'COMPEPSILON'][:] = parCat[
'compEpsilon'][0, :]
1364 inParams[
'COMPMEDDELTAAPER'][:] = parCat[
'compMedDeltaAper'][0, :]
1365 inParams[
'COMPGLOBALEPSILON'][:] = parCat[
'compGlobalEpsilon'][0, :]
1366 inParams[
'COMPEPSILONMAP'][:] = parCat[
'compEpsilonMap'][0, :]
1367 inParams[
'COMPEPSILONNSTARMAP'][:] = parCat[
'compEpsilonNStarMap'][0, :]
1368 inParams[
'COMPEPSILONCCDMAP'][:] = parCat[
'compEpsilonCcdMap'][0, :]
1369 inParams[
'COMPEPSILONCCDNSTARMAP'][:] = parCat[
'compEpsilonCcdNStarMap'][0, :]
1371 inSuperStar = np.zeros(parCat[
'superstarSize'][0, :], dtype=
'f8')
1372 inSuperStar[:, :, :, :] = parCat[
'superstar'][0, :].reshape(inSuperStar.shape)
1374 return (inParInfo, inParams, inSuperStar)
1376 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1378 Persist FGCM datasets through the butler.
1382 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1383 Fgcm Fit cycle object
1385 fgcmDatasetDict = {}
1388 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1390 parSchema = afwTable.Schema()
1393 lutFilterNameString = comma.join([n.decode(
'utf-8')
1394 for n
in parInfo[
'LUTFILTERNAMES'][0]])
1395 fitBandString = comma.join([n.decode(
'utf-8')
1396 for n
in parInfo[
'FITBANDS'][0]])
1398 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1399 lutFilterNameString, fitBandString)
1400 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1401 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1402 lutFilterNameString, fitBandString)
1404 fgcmDatasetDict[
'fgcmFitParameters'] = parCat
1409 flagStarSchema = self._makeFlagStarSchema()
1410 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1411 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1413 fgcmDatasetDict[
'fgcmFlaggedStars'] = flagStarCat
1416 if self.outputZeropoints:
1417 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
1418 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
1421 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1423 fgcmDatasetDict[
'fgcmZeropoints'] = zptCat
1428 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1430 fgcmDatasetDict[
'fgcmAtmosphereParameters'] = atmCat
1433 if self.outputStandards:
1434 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1436 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
1438 fgcmDatasetDict[
'fgcmStandardStars'] = stdCat
1440 return fgcmDatasetDict
1442 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1443 lutFilterNameString, fitBandString):
1445 Make the parameter persistence schema
1449 parInfo: `numpy.ndarray`
1450 Parameter information returned by fgcm
1451 pars: `numpy.ndarray`
1452 Parameter values returned by fgcm
1453 parSuperStarFlat: `numpy.array`
1454 Superstar flat values returned by fgcm
1455 lutFilterNameString: `str`
1456 Combined string of all the lutFilterNames
1457 fitBandString: `str`
1458 Combined string of all the fitBands
1462 parSchema: `afwTable.schema`
1465 parSchema = afwTable.Schema()
1468 parSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
1469 parSchema.addField(
'lutFilterNames', type=str, doc=
'LUT Filter names in parameter file',
1470 size=len(lutFilterNameString))
1471 parSchema.addField(
'fitBands', type=str, doc=
'Bands that were fit',
1472 size=len(fitBandString))
1473 parSchema.addField(
'lnTauUnit', type=np.float64, doc=
'Step units for ln(AOD)')
1474 parSchema.addField(
'lnTauSlopeUnit', type=np.float64,
1475 doc=
'Step units for ln(AOD) slope')
1476 parSchema.addField(
'alphaUnit', type=np.float64, doc=
'Step units for alpha')
1477 parSchema.addField(
'lnPwvUnit', type=np.float64, doc=
'Step units for ln(pwv)')
1478 parSchema.addField(
'lnPwvSlopeUnit', type=np.float64,
1479 doc=
'Step units for ln(pwv) slope')
1480 parSchema.addField(
'lnPwvQuadraticUnit', type=np.float64,
1481 doc=
'Step units for ln(pwv) quadratic term')
1482 parSchema.addField(
'lnPwvGlobalUnit', type=np.float64,
1483 doc=
'Step units for global ln(pwv) parameters')
1484 parSchema.addField(
'o3Unit', type=np.float64, doc=
'Step units for O3')
1485 parSchema.addField(
'qeSysUnit', type=np.float64, doc=
'Step units for mirror gray')
1486 parSchema.addField(
'filterOffsetUnit', type=np.float64, doc=
'Step units for filter offset')
1487 parSchema.addField(
'hasExternalPwv', type=np.int32, doc=
'Parameters fit using external pwv')
1488 parSchema.addField(
'hasExternalTau', type=np.int32, doc=
'Parameters fit using external tau')
1491 parSchema.addField(
'parAlpha', type=
'ArrayD', doc=
'Alpha parameter vector',
1492 size=pars[
'PARALPHA'].size)
1493 parSchema.addField(
'parO3', type=
'ArrayD', doc=
'O3 parameter vector',
1494 size=pars[
'PARO3'].size)
1495 parSchema.addField(
'parLnTauIntercept', type=
'ArrayD',
1496 doc=
'ln(Tau) intercept parameter vector',
1497 size=pars[
'PARLNTAUINTERCEPT'].size)
1498 parSchema.addField(
'parLnTauSlope', type=
'ArrayD',
1499 doc=
'ln(Tau) slope parameter vector',
1500 size=pars[
'PARLNTAUSLOPE'].size)
1501 parSchema.addField(
'parLnPwvIntercept', type=
'ArrayD', doc=
'ln(pwv) intercept parameter vector',
1502 size=pars[
'PARLNPWVINTERCEPT'].size)
1503 parSchema.addField(
'parLnPwvSlope', type=
'ArrayD', doc=
'ln(pwv) slope parameter vector',
1504 size=pars[
'PARLNPWVSLOPE'].size)
1505 parSchema.addField(
'parLnPwvQuadratic', type=
'ArrayD', doc=
'ln(pwv) quadratic parameter vector',
1506 size=pars[
'PARLNPWVQUADRATIC'].size)
1507 parSchema.addField(
'parQeSysIntercept', type=
'ArrayD', doc=
'Mirror gray intercept parameter vector',
1508 size=pars[
'PARQESYSINTERCEPT'].size)
1509 parSchema.addField(
'compQeSysSlope', type=
'ArrayD', doc=
'Mirror gray slope parameter vector',
1510 size=pars[0][
'COMPQESYSSLOPE'].size)
1511 parSchema.addField(
'parFilterOffset', type=
'ArrayD', doc=
'Filter offset parameter vector',
1512 size=pars[
'PARFILTEROFFSET'].size)
1513 parSchema.addField(
'parFilterOffsetFitFlag', type=
'ArrayI', doc=
'Filter offset parameter fit flag',
1514 size=pars[
'PARFILTEROFFSETFITFLAG'].size)
1515 parSchema.addField(
'parRetrievedLnPwvScale', type=np.float64,
1516 doc=
'Global scale for retrieved ln(pwv)')
1517 parSchema.addField(
'parRetrievedLnPwvOffset', type=np.float64,
1518 doc=
'Global offset for retrieved ln(pwv)')
1519 parSchema.addField(
'parRetrievedLnPwvNightlyOffset', type=
'ArrayD',
1520 doc=
'Nightly offset for retrieved ln(pwv)',
1521 size=pars[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1522 parSchema.addField(
'compAbsThroughput', type=
'ArrayD',
1523 doc=
'Absolute throughput (relative to transmission curves)',
1524 size=pars[
'COMPABSTHROUGHPUT'].size)
1525 parSchema.addField(
'compRefOffset', type=
'ArrayD',
1526 doc=
'Offset between reference stars and calibrated stars',
1527 size=pars[
'COMPREFOFFSET'].size)
1528 parSchema.addField(
'compRefSigma', type=
'ArrayD',
1529 doc=
'Width of reference star/calibrated star distribution',
1530 size=pars[
'COMPREFSIGMA'].size)
1531 parSchema.addField(
'compMirrorChromaticity', type=
'ArrayD',
1532 doc=
'Computed mirror chromaticity terms',
1533 size=pars[
'COMPMIRRORCHROMATICITY'].size)
1534 parSchema.addField(
'mirrorChromaticityPivot', type=
'ArrayD',
1535 doc=
'Mirror chromaticity pivot mjd',
1536 size=pars[
'MIRRORCHROMATICITYPIVOT'].size)
1537 parSchema.addField(
'compMedianSedSlope', type=
'ArrayD',
1538 doc=
'Computed median SED slope (per band)',
1539 size=pars[
'COMPMEDIANSEDSLOPE'].size)
1540 parSchema.addField(
'compAperCorrPivot', type=
'ArrayD', doc=
'Aperture correction pivot',
1541 size=pars[
'COMPAPERCORRPIVOT'].size)
1542 parSchema.addField(
'compAperCorrSlope', type=
'ArrayD', doc=
'Aperture correction slope',
1543 size=pars[
'COMPAPERCORRSLOPE'].size)
1544 parSchema.addField(
'compAperCorrSlopeErr', type=
'ArrayD', doc=
'Aperture correction slope error',
1545 size=pars[
'COMPAPERCORRSLOPEERR'].size)
1546 parSchema.addField(
'compAperCorrRange', type=
'ArrayD', doc=
'Aperture correction range',
1547 size=pars[
'COMPAPERCORRRANGE'].size)
1548 parSchema.addField(
'compModelErrExptimePivot', type=
'ArrayD', doc=
'Model error exptime pivot',
1549 size=pars[
'COMPMODELERREXPTIMEPIVOT'].size)
1550 parSchema.addField(
'compModelErrFwhmPivot', type=
'ArrayD', doc=
'Model error fwhm pivot',
1551 size=pars[
'COMPMODELERRFWHMPIVOT'].size)
1552 parSchema.addField(
'compModelErrSkyPivot', type=
'ArrayD', doc=
'Model error sky pivot',
1553 size=pars[
'COMPMODELERRSKYPIVOT'].size)
1554 parSchema.addField(
'compModelErrPars', type=
'ArrayD', doc=
'Model error parameters',
1555 size=pars[
'COMPMODELERRPARS'].size)
1556 parSchema.addField(
'compExpGray', type=
'ArrayD', doc=
'Computed exposure gray',
1557 size=pars[
'COMPEXPGRAY'].size)
1558 parSchema.addField(
'compVarGray', type=
'ArrayD', doc=
'Computed exposure variance',
1559 size=pars[
'COMPVARGRAY'].size)
1560 parSchema.addField(
'compExpDeltaMagBkg', type=
'ArrayD',
1561 doc=
'Computed exposure offset due to background',
1562 size=pars[
'COMPEXPDELTAMAGBKG'].size)
1563 parSchema.addField(
'compNGoodStarPerExp', type=
'ArrayI',
1564 doc=
'Computed number of good stars per exposure',
1565 size=pars[
'COMPNGOODSTARPEREXP'].size)
1566 parSchema.addField(
'compSigFgcm', type=
'ArrayD', doc=
'Computed sigma_fgcm (intrinsic repeatability)',
1567 size=pars[
'COMPSIGFGCM'].size)
1568 parSchema.addField(
'compSigmaCal', type=
'ArrayD', doc=
'Computed sigma_cal (systematic error floor)',
1569 size=pars[
'COMPSIGMACAL'].size)
1570 parSchema.addField(
'compRetrievedLnPwv', type=
'ArrayD', doc=
'Retrieved ln(pwv) (smoothed)',
1571 size=pars[
'COMPRETRIEVEDLNPWV'].size)
1572 parSchema.addField(
'compRetrievedLnPwvRaw', type=
'ArrayD', doc=
'Retrieved ln(pwv) (raw)',
1573 size=pars[
'COMPRETRIEVEDLNPWVRAW'].size)
1574 parSchema.addField(
'compRetrievedLnPwvFlag', type=
'ArrayI', doc=
'Retrieved ln(pwv) Flag',
1575 size=pars[
'COMPRETRIEVEDLNPWVFLAG'].size)
1576 parSchema.addField(
'compRetrievedTauNight', type=
'ArrayD', doc=
'Retrieved tau (per night)',
1577 size=pars[
'COMPRETRIEVEDTAUNIGHT'].size)
1578 parSchema.addField(
'compEpsilon', type=
'ArrayD',
1579 doc=
'Computed epsilon background offset per visit (nJy/arcsec2)',
1580 size=pars[
'COMPEPSILON'].size)
1581 parSchema.addField(
'compMedDeltaAper', type=
'ArrayD',
1582 doc=
'Median delta mag aper per visit',
1583 size=pars[
'COMPMEDDELTAAPER'].size)
1584 parSchema.addField(
'compGlobalEpsilon', type=
'ArrayD',
1585 doc=
'Computed epsilon bkg offset (global) (nJy/arcsec2)',
1586 size=pars[
'COMPGLOBALEPSILON'].size)
1587 parSchema.addField(
'compEpsilonMap', type=
'ArrayD',
1588 doc=
'Computed epsilon maps (nJy/arcsec2)',
1589 size=pars[
'COMPEPSILONMAP'].size)
1590 parSchema.addField(
'compEpsilonNStarMap', type=
'ArrayI',
1591 doc=
'Number of stars per pixel in computed epsilon maps',
1592 size=pars[
'COMPEPSILONNSTARMAP'].size)
1593 parSchema.addField(
'compEpsilonCcdMap', type=
'ArrayD',
1594 doc=
'Computed epsilon ccd maps (nJy/arcsec2)',
1595 size=pars[
'COMPEPSILONCCDMAP'].size)
1596 parSchema.addField(
'compEpsilonCcdNStarMap', type=
'ArrayI',
1597 doc=
'Number of stars per ccd bin in epsilon ccd maps',
1598 size=pars[
'COMPEPSILONCCDNSTARMAP'].size)
1600 parSchema.addField(
'superstarSize', type=
'ArrayI', doc=
'Superstar matrix size',
1602 parSchema.addField(
'superstar', type=
'ArrayD', doc=
'Superstar matrix (flattened)',
1603 size=parSuperStarFlat.size)
1607 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1608 lutFilterNameString, fitBandString):
1610 Make the FGCM parameter catalog for persistence
1614 parSchema: `lsst.afw.table.Schema`
1615 Parameter catalog schema
1616 pars: `numpy.ndarray`
1617 FGCM parameters to put into parCat
1618 parSuperStarFlat: `numpy.array`
1619 FGCM superstar flat array to put into parCat
1620 lutFilterNameString: `str`
1621 Combined string of all the lutFilterNames
1622 fitBandString: `str`
1623 Combined string of all the fitBands
1627 parCat: `afwTable.BasicCatalog`
1628 Atmosphere
and instrumental model parameter catalog
for persistence
1631 parCat = afwTable.BaseCatalog(parSchema)
1636 rec = parCat.addNew()
1639 rec[
'nCcd'] = parInfo[
'NCCD']
1640 rec[
'lutFilterNames'] = lutFilterNameString
1641 rec[
'fitBands'] = fitBandString
1643 rec[
'hasExternalPwv'] = 0
1644 rec[
'hasExternalTau'] = 0
1648 scalarNames = [
'parRetrievedLnPwvScale',
'parRetrievedLnPwvOffset']
1650 arrNames = [
'parAlpha',
'parO3',
'parLnTauIntercept',
'parLnTauSlope',
1651 'parLnPwvIntercept',
'parLnPwvSlope',
'parLnPwvQuadratic',
1652 'parQeSysIntercept',
'compQeSysSlope',
1653 'parRetrievedLnPwvNightlyOffset',
'compAperCorrPivot',
1654 'parFilterOffset',
'parFilterOffsetFitFlag',
1655 'compAbsThroughput',
'compRefOffset',
'compRefSigma',
1656 'compMirrorChromaticity',
'mirrorChromaticityPivot',
1657 'compAperCorrSlope',
'compAperCorrSlopeErr',
'compAperCorrRange',
1658 'compModelErrExptimePivot',
'compModelErrFwhmPivot',
1659 'compModelErrSkyPivot',
'compModelErrPars',
1660 'compExpGray',
'compVarGray',
'compNGoodStarPerExp',
'compSigFgcm',
1661 'compSigmaCal',
'compExpDeltaMagBkg',
'compMedianSedSlope',
1662 'compRetrievedLnPwv',
'compRetrievedLnPwvRaw',
'compRetrievedLnPwvFlag',
1663 'compRetrievedTauNight',
'compEpsilon',
'compMedDeltaAper',
1664 'compGlobalEpsilon',
'compEpsilonMap',
'compEpsilonNStarMap',
1665 'compEpsilonCcdMap',
'compEpsilonCcdNStarMap']
1667 for scalarName
in scalarNames:
1668 rec[scalarName] = pars[scalarName.upper()]
1670 for arrName
in arrNames:
1671 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1674 rec[
'superstarSize'][:] = parSuperStarFlat.shape
1675 rec[
'superstar'][:] = parSuperStarFlat.ravel()
1679 def _makeFlagStarSchema(self):
1681 Make the flagged-stars schema
1685 flagStarSchema: `lsst.afw.table.Schema`
1688 flagStarSchema = afwTable.Schema()
1690 flagStarSchema.addField('objId', type=np.int32, doc=
'FGCM object id')
1691 flagStarSchema.addField(
'objFlag', type=np.int32, doc=
'FGCM object flag')
1693 return flagStarSchema
1695 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1697 Make the flagged star catalog for persistence
1701 flagStarSchema: `lsst.afw.table.Schema`
1703 flagStarStruct: `numpy.ndarray`
1704 Flagged star structure
from fgcm
1708 flagStarCat: `lsst.afw.table.BaseCatalog`
1709 Flagged star catalog
for persistence
1712 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1713 flagStarCat.resize(flagStarStruct.size)
1715 flagStarCat['objId'][:] = flagStarStruct[
'OBJID']
1716 flagStarCat[
'objFlag'][:] = flagStarStruct[
'OBJFLAG']
def extractReferenceMags(refStars, bands, filterMap)
def makeStdSchema(nBands)
def makeAtmCat(atmSchema, atmStruct)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
def translateFgcmLut(lutCat, physicalFilterMap)
def makeZptCat(zptSchema, zpStruct)
def makeStdCat(stdSchema, stdStruct, goodBands)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def translateVisitCatalog(visitCat)