23 """Perform a single fit cycle of FGCM.
25 This task runs a single "fit cycle" of fgcm. Prior to running this task
26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27 look-up-table) and fgcmBuildStars (to extract visits and star observations
30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32 be inspected to set parameters for outlier rejection on the following
33 cycle. Please see the fgcmcal Cookbook for details.
42 import lsst.pex.config
as pexConfig
43 import lsst.pipe.base
as pipeBase
44 import lsst.afw.table
as afwTable
46 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
47 from .utilities
import extractReferenceMags
48 from .utilities
import computeCcdOffsets, makeZptSchema, makeZptCat
49 from .utilities
import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50 from .sedterms
import SedboundarytermDict, SedtermDict
54 __all__ = [
'FgcmFitCycleConfig',
'FgcmFitCycleTask',
'FgcmFitCycleRunner']
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc=
"Bands to run calibration",
65 fitFlag = pexConfig.ListField(
66 doc=(
"Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
73 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
74 "It will be removed after v20. Use fitBands instead."),
76 fitBands = pexConfig.ListField(
77 doc=(
"Bands to use in atmospheric fit. The bands not listed here will have "
78 "the atmosphere constrained from the 'fitBands' on the same night. "
79 "Must be a subset of `config.bands`"),
83 requiredFlag = pexConfig.ListField(
84 doc=(
"Flag for which bands are required for a star to be considered a calibration "
85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
86 "be same length as config.bands, and matched band-by-band."),
90 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
91 "It will be removed after v20. Use requiredBands instead."),
93 requiredBands = pexConfig.ListField(
94 doc=(
"Bands that are required for a star to be considered a calibration star. "
95 "Must be a subset of `config.bands`"),
99 filterMap = pexConfig.DictField(
100 doc=
"Mapping from 'filterName' to band.",
105 doReferenceCalibration = pexConfig.Field(
106 doc=
"Use reference catalog as additional constraint on calibration",
110 refStarSnMin = pexConfig.Field(
111 doc=
"Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
115 refStarOutlierNSig = pexConfig.Field(
116 doc=(
"Number of sigma compared to average mag for reference star to be considered an outlier. "
117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
121 applyRefStarColorCuts = pexConfig.Field(
122 doc=
"Apply color cuts to reference stars?",
126 nCore = pexConfig.Field(
127 doc=
"Number of cores to use",
131 nStarPerRun = pexConfig.Field(
132 doc=
"Number of stars to run in each chunk",
136 nExpPerRun = pexConfig.Field(
137 doc=
"Number of exposures to run in each chunk",
141 reserveFraction = pexConfig.Field(
142 doc=
"Fraction of stars to reserve for testing",
146 freezeStdAtmosphere = pexConfig.Field(
147 doc=
"Freeze atmosphere parameters to standard (for testing)",
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc=
"Precompute superstar flat for initial cycle",
156 superStarSubCcd = pexConfig.Field(
157 doc=
"Compute superstar flat on sub-ccd scale",
161 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
162 "It will be removed after v20. Use superStarSubCcdDict instead."),
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=(
"Per-band specification on whether to compute superstar flat on sub-ccd scale. "
166 "Must have one entry per band."),
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=(
"Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
173 "Global default is first-order polynomials, and should be overridden "
174 "on a camera-by-camera basis depending on the ISR."),
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=(
"Should the sub-ccd superstar chebyshev matrix be triangular to "
180 "suppress high-order cross terms?"),
184 superStarSigmaClip = pexConfig.Field(
185 doc=
"Number of sigma to clip outliers when selecting for superstar flats",
189 ccdGraySubCcd = pexConfig.Field(
190 doc=
"Compute CCD gray terms on sub-ccd scale",
194 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
195 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
197 ccdGraySubCcdDict = pexConfig.DictField(
198 doc=(
"Per-band specification on whether to compute achromatic per-ccd residual "
199 "('ccd gray') on a sub-ccd scale."),
204 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205 doc=
"Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
209 ccdGraySubCcdTriangular = pexConfig.Field(
210 doc=(
"Should the sub-ccd gray chebyshev matrix be triangular to "
211 "suppress high-order cross terms?"),
215 cycleNumber = pexConfig.Field(
216 doc=(
"FGCM fit cycle number. This is automatically incremented after each run "
217 "and stage of outlier rejection. See cookbook for details."),
221 isFinalCycle = pexConfig.Field(
222 doc=(
"Is this the final cycle of the fitting? Will automatically compute final "
223 "selection of stars and photometric exposures, and will output zeropoints "
224 "and standard stars for use in fgcmOutputProducts"),
228 maxIterBeforeFinalCycle = pexConfig.Field(
229 doc=(
"Maximum fit iterations, prior to final cycle. The number of iterations "
230 "will always be 0 in the final cycle for cleanup and final selection."),
234 deltaMagBkgOffsetPercentile = pexConfig.Field(
235 doc=(
"Percentile brightest stars on a visit/ccd to use to compute net "
236 "offset from local background subtraction."),
240 deltaMagBkgPerCcd = pexConfig.Field(
241 doc=(
"Compute net offset from local background subtraction per-ccd? "
242 "Otherwise, use computation per visit."),
246 utBoundary = pexConfig.Field(
247 doc=
"Boundary (in UTC) from day-to-day",
251 washMjds = pexConfig.ListField(
252 doc=
"Mirror wash MJDs",
256 epochMjds = pexConfig.ListField(
257 doc=
"Epoch boundaries in MJD",
261 minObsPerBand = pexConfig.Field(
262 doc=
"Minimum good observations per band",
268 latitude = pexConfig.Field(
269 doc=
"Observatory latitude",
273 brightObsGrayMax = pexConfig.Field(
274 doc=
"Maximum gray extinction to be considered bright observation",
278 minStarPerCcd = pexConfig.Field(
279 doc=(
"Minimum number of good stars per CCD to be used in calibration fit. "
280 "CCDs with fewer stars will have their calibration estimated from other "
281 "CCDs in the same visit, with zeropoint error increased accordingly."),
285 minCcdPerExp = pexConfig.Field(
286 doc=(
"Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
287 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
291 maxCcdGrayErr = pexConfig.Field(
292 doc=
"Maximum error on CCD gray offset to be considered photometric",
296 minStarPerExp = pexConfig.Field(
297 doc=(
"Minimum number of good stars per exposure/visit to be used in calibration fit. "
298 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
302 minExpPerNight = pexConfig.Field(
303 doc=
"Minimum number of good exposures/visits to consider a partly photometric night",
307 expGrayInitialCut = pexConfig.Field(
308 doc=(
"Maximum exposure/visit gray value for initial selection of possible photometric "
313 expGrayPhotometricCut = pexConfig.ListField(
314 doc=(
"Maximum (negative) exposure gray for a visit to be considered photometric. "
315 "Must be same length as config.bands, and matched band-by-band."),
319 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
320 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
322 expGrayPhotometricCutDict = pexConfig.DictField(
323 doc=(
"Per-band specification on maximum (negative) achromatic exposure residual "
324 "('gray term') for a visit to be considered photometric. Must have one "
325 "entry per band. Broad-band filters should be -0.05."),
330 expGrayHighCut = pexConfig.ListField(
331 doc=(
"Maximum (positive) exposure gray for a visit to be considered photometric. "
332 "Must be same length as config.bands, and matched band-by-band."),
336 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
337 "It will be removed after v20. Use expGrayHighCutDict instead."),
339 expGrayHighCutDict = pexConfig.DictField(
340 doc=(
"Per-band specification on maximum (positive) achromatic exposure residual "
341 "('gray term') for a visit to be considered photometric. Must have one "
342 "entry per band. Broad-band filters should be 0.2."),
347 expGrayRecoverCut = pexConfig.Field(
348 doc=(
"Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
349 "Visits with more gray extinction will only get CCD zeropoints if there are "
350 "sufficient star observations (minStarPerCcd) on that CCD."),
354 expVarGrayPhotometricCut = pexConfig.Field(
355 doc=
"Maximum exposure variance to be considered possibly photometric",
359 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
360 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
362 expVarGrayPhotometricCutDict = pexConfig.DictField(
363 doc=(
"Per-band specification on maximum exposure variance to be considered possibly "
364 "photometric. Must have one entry per band. Broad-band filters should be "
370 expGrayErrRecoverCut = pexConfig.Field(
371 doc=(
"Maximum exposure gray error to be able to recover bad ccds via interpolation. "
372 "Visits with more gray variance will only get CCD zeropoints if there are "
373 "sufficient star observations (minStarPerCcd) on that CCD."),
377 aperCorrFitNBins = pexConfig.Field(
378 doc=(
"Number of aperture bins used in aperture correction fit. When set to 0"
379 "no fit will be performed, and the config.aperCorrInputSlopes will be "
380 "used if available."),
384 aperCorrInputSlopes = pexConfig.ListField(
385 doc=(
"Aperture correction input slope parameters. These are used on the first "
386 "fit iteration, and aperture correction parameters will be updated from "
387 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
388 "when there is insufficient data to fit the parameters (e.g. tract mode). "
389 "If set, must be same length as config.bands, and matched band-by-band."),
393 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
394 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
396 aperCorrInputSlopeDict = pexConfig.DictField(
397 doc=(
"Per-band specification of aperture correction input slope parameters. These "
398 "are used on the first fit iteration, and aperture correction parameters will "
399 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
400 "to set this when there is insufficient data to fit the parameters (e.g. "
406 sedFudgeFactors = pexConfig.ListField(
407 doc=(
"Fudge factors for computing linear SED from colors. Must be same length as "
408 "config.bands, and matched band-by-band."),
412 deprecated=(
"This field has been deprecated and will be removed after v20. "
413 "Please use sedSlopeTermMap and sedSlopeMap."),
415 sedboundaryterms = pexConfig.ConfigField(
416 doc=
"Mapping from bands to SED boundary term names used is sedterms.",
417 dtype=SedboundarytermDict,
419 sedterms = pexConfig.ConfigField(
420 doc=
"Mapping from terms to bands for fgcm linear SED approximations.",
423 sigFgcmMaxErr = pexConfig.Field(
424 doc=
"Maximum mag error for fitting sigma_FGCM",
428 sigFgcmMaxEGray = pexConfig.ListField(
429 doc=(
"Maximum (absolute) gray value for observation in sigma_FGCM. "
430 "May be 1 element (same for all bands) or the same length as config.bands."),
434 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
435 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
437 sigFgcmMaxEGrayDict = pexConfig.DictField(
438 doc=(
"Per-band specification for maximum (absolute) achromatic residual (gray value) "
439 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
445 ccdGrayMaxStarErr = pexConfig.Field(
446 doc=(
"Maximum error on a star observation to use in ccd gray (achromatic residual) "
451 approxThroughput = pexConfig.ListField(
452 doc=(
"Approximate overall throughput at start of calibration observations. "
453 "May be 1 element (same for all bands) or the same length as config.bands."),
457 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
458 "It will be removed after v20. Use approxThroughputDict instead."),
460 approxThroughputDict = pexConfig.DictField(
461 doc=(
"Per-band specification of the approximate overall throughput at the start of "
462 "calibration observations. Must have one entry per band. Typically should "
468 sigmaCalRange = pexConfig.ListField(
469 doc=
"Allowed range for systematic error floor estimation",
471 default=(0.001, 0.003),
473 sigmaCalFitPercentile = pexConfig.ListField(
474 doc=
"Magnitude percentile range to fit systematic error floor",
476 default=(0.05, 0.15),
478 sigmaCalPlotPercentile = pexConfig.ListField(
479 doc=
"Magnitude percentile range to plot systematic error floor",
481 default=(0.05, 0.95),
483 sigma0Phot = pexConfig.Field(
484 doc=
"Systematic error floor for all zeropoints",
488 mapLongitudeRef = pexConfig.Field(
489 doc=
"Reference longitude for plotting maps",
493 mapNSide = pexConfig.Field(
494 doc=
"Healpix nside for plotting maps",
498 outfileBase = pexConfig.Field(
499 doc=
"Filename start for plot output files",
503 starColorCuts = pexConfig.ListField(
504 doc=
"Encoded star-color cuts (to be cleaned up)",
506 default=(
"NO_DATA",),
508 colorSplitIndices = pexConfig.ListField(
509 doc=
"Band indices to use to split stars by color",
513 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
514 "It will be removed after v20. Use colorSplitBands instead."),
516 colorSplitBands = pexConfig.ListField(
517 doc=
"Band names to use to split stars by color. Must have 2 entries.",
522 modelMagErrors = pexConfig.Field(
523 doc=
"Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
527 useQuadraticPwv = pexConfig.Field(
528 doc=
"Model PWV with a quadratic term for variation through the night?",
532 instrumentParsPerBand = pexConfig.Field(
533 doc=(
"Model instrumental parameters per band? "
534 "Otherwise, instrumental parameters (QE changes with time) are "
535 "shared among all bands."),
539 instrumentSlopeMinDeltaT = pexConfig.Field(
540 doc=(
"Minimum time change (in days) between observations to use in constraining "
541 "instrument slope."),
545 fitMirrorChromaticity = pexConfig.Field(
546 doc=
"Fit (intraband) mirror chromatic term?",
550 coatingMjds = pexConfig.ListField(
551 doc=
"Mirror coating dates in MJD",
555 outputStandardsBeforeFinalCycle = pexConfig.Field(
556 doc=
"Output standard stars prior to final cycle? Used in debugging.",
560 outputZeropointsBeforeFinalCycle = pexConfig.Field(
561 doc=
"Output standard stars prior to final cycle? Used in debugging.",
565 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
566 doc=(
"Use star repeatability (instead of exposures) for computing photometric "
567 "cuts? Recommended for tract mode or bands with few exposures. "
568 "May be 1 element (same for all bands) or the same length as config.bands."),
572 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. "
573 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
575 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
576 doc=(
"Per-band specification on whether to use star repeatability (instead of exposures) "
577 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
582 autoPhotometricCutNSig = pexConfig.Field(
583 doc=(
"Number of sigma for automatic computation of (low) photometric cut. "
584 "Cut is based on exposure gray width (per band), unless "
585 "useRepeatabilityForExpGrayCuts is set, in which case the star "
586 "repeatability is used (also per band)."),
590 autoHighCutNSig = pexConfig.Field(
591 doc=(
"Number of sigma for automatic computation of (high) outlier cut. "
592 "Cut is based on exposure gray width (per band), unless "
593 "useRepeatabilityForExpGrayCuts is set, in which case the star "
594 "repeatability is used (also per band)."),
598 quietMode = pexConfig.Field(
599 doc=
"Be less verbose with logging.",
610 for band
in self.fitBands:
611 if band
not in self.bands:
612 msg =
'fitBand %s not in bands' % (band)
613 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
614 for band
in self.requiredBands:
615 if band
not in self.bands:
616 msg =
'requiredBand %s not in bands' % (band)
617 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
618 for band
in self.colorSplitBands:
619 if band
not in self.bands:
620 msg =
'colorSplitBand %s not in bands' % (band)
621 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
622 for band
in self.bands:
623 if band
not in self.superStarSubCcdDict:
624 msg =
'band %s not in superStarSubCcdDict' % (band)
625 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
627 if band
not in self.ccdGraySubCcdDict:
628 msg =
'band %s not in ccdGraySubCcdDict' % (band)
629 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
631 if band
not in self.expGrayPhotometricCutDict:
632 msg =
'band %s not in expGrayPhotometricCutDict' % (band)
633 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
635 if band
not in self.expGrayHighCutDict:
636 msg =
'band %s not in expGrayHighCutDict' % (band)
637 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
639 if band
not in self.expVarGrayPhotometricCutDict:
640 msg =
'band %s not in expVarGrayPhotometricCutDict' % (band)
641 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
643 if band
not in self.sigFgcmMaxEGrayDict:
644 msg =
'band %s not in sigFgcmMaxEGrayDict' % (band)
645 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
647 if band
not in self.approxThroughputDict:
648 msg =
'band %s not in approxThroughputDict' % (band)
649 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
651 if band
not in self.useRepeatabilityForExpGrayCutsDict:
652 msg =
'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
653 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
658 """Subclass of TaskRunner for fgcmFitCycleTask
660 fgcmFitCycleTask.run() takes one argument, the butler, and uses
661 stars and visits previously extracted from dataRefs by
663 This Runner does not perform any dataRef parallelization, but the FGCM
664 code called by the Task uses python multiprocessing (see the "ncores"
671 Return a list with one element, the butler.
673 return [parsedCmd.butler]
679 butler: `lsst.daf.persistence.Butler`
683 exitStatus: `list` with `pipeBase.Struct`
684 exitStatus (0: success; 1: failure)
687 task = self.TaskClass(config=self.config, log=self.log)
691 task.runDataRef(butler)
694 task.runDataRef(butler)
695 except Exception
as e:
697 task.log.fatal(
"Failed: %s" % e)
698 if not isinstance(e, pipeBase.TaskError):
699 traceback.print_exc(file=sys.stderr)
701 task.writeMetadata(butler)
704 return [pipeBase.Struct(exitStatus=exitStatus)]
708 Run the task, with no multiprocessing
712 parsedCmd: ArgumentParser parsed command line
717 if self.precall(parsedCmd):
720 resultList = self(targetList[0])
727 Run Single fit cycle for FGCM global calibration
730 ConfigClass = FgcmFitCycleConfig
731 RunnerClass = FgcmFitCycleRunner
732 _DefaultName =
"fgcmFitCycle"
736 Instantiate an fgcmFitCycle.
740 butler : `lsst.daf.persistence.Butler`
743 pipeBase.CmdLineTask.__init__(self, **kwargs)
746 def _getMetadataName(self):
752 Run a single fit cycle for FGCM
756 butler: `lsst.daf.persistence.Butler`
762 """Write the configuration used for processing the data, or check that an existing
763 one is equal to the new one if present. This is an override of the regular
764 version from pipe_base that knows about fgcmcycle.
768 butler : `lsst.daf.persistence.Butler`
769 Data butler used to write the config. The config is written to dataset type
770 `CmdLineTask._getConfigName`.
771 clobber : `bool`, optional
772 A boolean flag that controls what happens if a config already has been saved:
773 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
774 - `False`: raise `TaskError` if this config does not match the existing config.
775 doBackup : `bool`, optional
776 Set to `True` to backup the config files if clobbering.
778 configName = self._getConfigName()
779 if configName
is None:
782 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
783 elif butler.datasetExists(configName, write=
True, fgcmcycle=self.config.cycleNumber):
786 oldConfig = butler.get(configName, immediate=
True, fgcmcycle=self.config.cycleNumber)
787 except Exception
as exc:
788 raise type(exc)(
"Unable to read stored config file %s (%s); consider using --clobber-config" %
791 def logConfigMismatch(msg):
792 self.log.fatal(
"Comparing configuration: %s", msg)
794 if not self.config.compare(oldConfig, shortcut=
False, output=logConfigMismatch):
795 raise pipeBase.TaskError(
796 (
"Config does not match existing task config %r on disk; tasks configurations " +
797 "must be consistent within the same output repo (override with --clobber-config)") %
800 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
802 def _fgcmFitCycle(self, butler):
808 butler: `lsst.daf.persistence.Butler`
814 self.
maxIter = self.config.maxIterBeforeFinalCycle
819 if self.config.isFinalCycle:
828 camera = butler.get(
'camera')
833 lutCat = butler.get(
'fgcmLookUpTable')
834 fgcmLut, lutIndexVals, lutStd =
translateFgcmLut(lutCat, dict(self.config.filterMap))
840 visitCat = butler.get(
'fgcmVisitCatalog')
848 noFitsDict = {
'lutIndex': lutIndexVals,
850 'expInfo': fgcmExpInfo,
851 'ccdOffsets': ccdOffsets}
854 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
855 noFitsDict=noFitsDict, noOutput=
True)
858 if (fgcmFitCycle.initialCycle):
860 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
865 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
871 lastCycle = configDict[
'cycleNumber'] - 1
874 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
876 starObs = butler.get(
'fgcmStarObservations')
877 starIds = butler.get(
'fgcmStarIds')
878 starIndices = butler.get(
'fgcmStarIndices')
881 if butler.datasetExists(
'fgcmFlaggedStars', fgcmcycle=lastCycle):
882 flaggedStars = butler.get(
'fgcmFlaggedStars', fgcmcycle=lastCycle)
883 flagId = flaggedStars[
'objId'][:]
884 flagFlag = flaggedStars[
'objFlag'][:]
890 if self.config.doReferenceCalibration:
891 refStars = butler.get(
'fgcmReferenceStars')
895 self.config.filterMap)
896 refId = refStars[
'fgcm_id'][:]
906 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'], starObs[
'visit'][starIndices[
'obsIndex']])
918 conv = starObs[0][
'ra'].asDegrees() / float(starObs[0][
'ra'])
920 fgcmStars.loadStars(fgcmPars,
921 starObs[
'visit'][starIndices[
'obsIndex']],
922 starObs[
'ccd'][starIndices[
'obsIndex']],
923 starObs[
'ra'][starIndices[
'obsIndex']] * conv,
924 starObs[
'dec'][starIndices[
'obsIndex']] * conv,
925 starObs[
'instMag'][starIndices[
'obsIndex']],
926 starObs[
'instMagErr'][starIndices[
'obsIndex']],
927 fgcmExpInfo[
'FILTERNAME'][visitIndex],
928 starIds[
'fgcm_id'][:],
931 starIds[
'obsArrIndex'][:],
933 obsX=starObs[
'x'][starIndices[
'obsIndex']],
934 obsY=starObs[
'y'][starIndices[
'obsIndex']],
935 obsDeltaMagBkg=starObs[
'deltaMagBkg'][starIndices[
'obsIndex']],
936 psfCandidate=starObs[
'psf_candidate'][starIndices[
'obsIndex']],
957 fgcmFitCycle.setLUT(fgcmLut)
958 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
959 fgcmFitCycle.setPars(fgcmPars)
962 fgcmFitCycle.finishSetup()
976 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i])
for
977 i, b
in enumerate(self.config.bands)}
978 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i])
for
979 i, band
in enumerate(self.config.bands)}
981 outConfig = copy.copy(self.config)
982 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
983 precomputeSuperStarInitialCycle=
False,
984 freezeStdAtmosphere=
False,
985 expGrayPhotometricCutDict=updatedPhotometricCutDict,
986 expGrayHighCutDict=updatedHighCutDict)
987 configFileName =
'%s_cycle%02d_config.py' % (outConfig.outfileBase,
988 outConfig.cycleNumber)
989 outConfig.save(configFileName)
991 if self.config.isFinalCycle == 1:
993 self.log.info(
"Everything is in place to run fgcmOutputProducts.py")
995 self.log.info(
"Saved config for next cycle to %s" % (configFileName))
996 self.log.info(
"Be sure to look at:")
997 self.log.info(
" config.expGrayPhotometricCut")
998 self.log.info(
" config.expGrayHighCut")
999 self.log.info(
"If you are satisfied with the fit, please set:")
1000 self.log.info(
" config.isFinalCycle = True")
1002 def _checkDatasetsExist(self, butler):
1004 Check if necessary datasets exist to run fgcmFitCycle
1008 butler: `lsst.daf.persistence.Butler`
1013 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1014 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1015 If cycleNumber > 0, then also checks for fgcmFitParameters,
1019 if not butler.datasetExists(
'fgcmVisitCatalog'):
1020 raise RuntimeError(
"Could not find fgcmVisitCatalog in repo!")
1021 if not butler.datasetExists(
'fgcmStarObservations'):
1022 raise RuntimeError(
"Could not find fgcmStarObservations in repo!")
1023 if not butler.datasetExists(
'fgcmStarIds'):
1024 raise RuntimeError(
"Could not find fgcmStarIds in repo!")
1025 if not butler.datasetExists(
'fgcmStarIndices'):
1026 raise RuntimeError(
"Could not find fgcmStarIndices in repo!")
1027 if not butler.datasetExists(
'fgcmLookUpTable'):
1028 raise RuntimeError(
"Could not find fgcmLookUpTable in repo!")
1031 if (self.config.cycleNumber > 0):
1032 if not butler.datasetExists(
'fgcmFitParameters',
1033 fgcmcycle=self.config.cycleNumber-1):
1034 raise RuntimeError(
"Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1035 (self.config.cycleNumber-1))
1036 if not butler.datasetExists(
'fgcmFlaggedStars',
1037 fgcmcycle=self.config.cycleNumber-1):
1038 raise RuntimeError(
"Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1039 (self.config.cycleNumber-1))
1042 if self.config.doReferenceCalibration:
1043 if not butler.datasetExists(
'fgcmReferenceStars'):
1044 raise RuntimeError(
"Could not find fgcmReferenceStars in repo, and "
1045 "doReferenceCalibration is True.")
1047 def _loadParameters(self, butler):
1049 Load FGCM parameters from a previous fit cycle
1053 butler: `lsst.daf.persistence.Butler`
1057 inParInfo: `numpy.ndarray`
1058 Numpy array parameter information formatted for input to fgcm
1059 inParameters: `numpy.ndarray`
1060 Numpy array parameter values formatted for input to fgcm
1061 inSuperStar: `numpy.array`
1062 Superstar flat formatted for input to fgcm
1066 parCat = butler.get(
'fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1068 parLutFilterNames = np.array(parCat[0][
'lutFilterNames'].split(
','))
1069 parFitBands = np.array(parCat[0][
'fitBands'].split(
','))
1071 inParInfo = np.zeros(1, dtype=[(
'NCCD',
'i4'),
1072 (
'LUTFILTERNAMES', parLutFilterNames.dtype.str,
1073 (parLutFilterNames.size, )),
1074 (
'FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1075 (
'LNTAUUNIT',
'f8'),
1076 (
'LNTAUSLOPEUNIT',
'f8'),
1077 (
'ALPHAUNIT',
'f8'),
1078 (
'LNPWVUNIT',
'f8'),
1079 (
'LNPWVSLOPEUNIT',
'f8'),
1080 (
'LNPWVQUADRATICUNIT',
'f8'),
1081 (
'LNPWVGLOBALUNIT',
'f8'),
1083 (
'QESYSUNIT',
'f8'),
1084 (
'FILTEROFFSETUNIT',
'f8'),
1085 (
'HASEXTERNALPWV',
'i2'),
1086 (
'HASEXTERNALTAU',
'i2')])
1087 inParInfo[
'NCCD'] = parCat[
'nCcd']
1088 inParInfo[
'LUTFILTERNAMES'][:] = parLutFilterNames
1089 inParInfo[
'FITBANDS'][:] = parFitBands
1090 inParInfo[
'HASEXTERNALPWV'] = parCat[
'hasExternalPwv']
1091 inParInfo[
'HASEXTERNALTAU'] = parCat[
'hasExternalTau']
1093 inParams = np.zeros(1, dtype=[(
'PARALPHA',
'f8', (parCat[
'parAlpha'].size, )),
1094 (
'PARO3',
'f8', (parCat[
'parO3'].size, )),
1095 (
'PARLNTAUINTERCEPT',
'f8',
1096 (parCat[
'parLnTauIntercept'].size, )),
1097 (
'PARLNTAUSLOPE',
'f8',
1098 (parCat[
'parLnTauSlope'].size, )),
1099 (
'PARLNPWVINTERCEPT',
'f8',
1100 (parCat[
'parLnPwvIntercept'].size, )),
1101 (
'PARLNPWVSLOPE',
'f8',
1102 (parCat[
'parLnPwvSlope'].size, )),
1103 (
'PARLNPWVQUADRATIC',
'f8',
1104 (parCat[
'parLnPwvQuadratic'].size, )),
1105 (
'PARQESYSINTERCEPT',
'f8',
1106 (parCat[
'parQeSysIntercept'].size, )),
1107 (
'COMPQESYSSLOPE',
'f8',
1108 (parCat[
'compQeSysSlope'].size, )),
1109 (
'PARFILTEROFFSET',
'f8',
1110 (parCat[
'parFilterOffset'].size, )),
1111 (
'PARFILTEROFFSETFITFLAG',
'i2',
1112 (parCat[
'parFilterOffsetFitFlag'].size, )),
1113 (
'PARRETRIEVEDLNPWVSCALE',
'f8'),
1114 (
'PARRETRIEVEDLNPWVOFFSET',
'f8'),
1115 (
'PARRETRIEVEDLNPWVNIGHTLYOFFSET',
'f8',
1116 (parCat[
'parRetrievedLnPwvNightlyOffset'].size, )),
1117 (
'COMPABSTHROUGHPUT',
'f8',
1118 (parCat[
'compAbsThroughput'].size, )),
1119 (
'COMPREFOFFSET',
'f8',
1120 (parCat[
'compRefOffset'].size, )),
1121 (
'COMPREFSIGMA',
'f8',
1122 (parCat[
'compRefSigma'].size, )),
1123 (
'COMPMIRRORCHROMATICITY',
'f8',
1124 (parCat[
'compMirrorChromaticity'].size, )),
1125 (
'MIRRORCHROMATICITYPIVOT',
'f8',
1126 (parCat[
'mirrorChromaticityPivot'].size, )),
1127 (
'COMPMEDIANSEDSLOPE',
'f8',
1128 (parCat[
'compMedianSedSlope'].size, )),
1129 (
'COMPAPERCORRPIVOT',
'f8',
1130 (parCat[
'compAperCorrPivot'].size, )),
1131 (
'COMPAPERCORRSLOPE',
'f8',
1132 (parCat[
'compAperCorrSlope'].size, )),
1133 (
'COMPAPERCORRSLOPEERR',
'f8',
1134 (parCat[
'compAperCorrSlopeErr'].size, )),
1135 (
'COMPAPERCORRRANGE',
'f8',
1136 (parCat[
'compAperCorrRange'].size, )),
1137 (
'COMPMODELERREXPTIMEPIVOT',
'f8',
1138 (parCat[
'compModelErrExptimePivot'].size, )),
1139 (
'COMPMODELERRFWHMPIVOT',
'f8',
1140 (parCat[
'compModelErrFwhmPivot'].size, )),
1141 (
'COMPMODELERRSKYPIVOT',
'f8',
1142 (parCat[
'compModelErrSkyPivot'].size, )),
1143 (
'COMPMODELERRPARS',
'f8',
1144 (parCat[
'compModelErrPars'].size, )),
1145 (
'COMPEXPGRAY',
'f8',
1146 (parCat[
'compExpGray'].size, )),
1147 (
'COMPVARGRAY',
'f8',
1148 (parCat[
'compVarGray'].size, )),
1149 (
'COMPEXPDELTAMAGBKG',
'f8',
1150 (parCat[
'compExpDeltaMagBkg'].size, )),
1151 (
'COMPNGOODSTARPEREXP',
'i4',
1152 (parCat[
'compNGoodStarPerExp'].size, )),
1153 (
'COMPSIGFGCM',
'f8',
1154 (parCat[
'compSigFgcm'].size, )),
1155 (
'COMPSIGMACAL',
'f8',
1156 (parCat[
'compSigmaCal'].size, )),
1157 (
'COMPRETRIEVEDLNPWV',
'f8',
1158 (parCat[
'compRetrievedLnPwv'].size, )),
1159 (
'COMPRETRIEVEDLNPWVRAW',
'f8',
1160 (parCat[
'compRetrievedLnPwvRaw'].size, )),
1161 (
'COMPRETRIEVEDLNPWVFLAG',
'i2',
1162 (parCat[
'compRetrievedLnPwvFlag'].size, )),
1163 (
'COMPRETRIEVEDTAUNIGHT',
'f8',
1164 (parCat[
'compRetrievedTauNight'].size, ))])
1166 inParams[
'PARALPHA'][:] = parCat[
'parAlpha'][0, :]
1167 inParams[
'PARO3'][:] = parCat[
'parO3'][0, :]
1168 inParams[
'PARLNTAUINTERCEPT'][:] = parCat[
'parLnTauIntercept'][0, :]
1169 inParams[
'PARLNTAUSLOPE'][:] = parCat[
'parLnTauSlope'][0, :]
1170 inParams[
'PARLNPWVINTERCEPT'][:] = parCat[
'parLnPwvIntercept'][0, :]
1171 inParams[
'PARLNPWVSLOPE'][:] = parCat[
'parLnPwvSlope'][0, :]
1172 inParams[
'PARLNPWVQUADRATIC'][:] = parCat[
'parLnPwvQuadratic'][0, :]
1173 inParams[
'PARQESYSINTERCEPT'][:] = parCat[
'parQeSysIntercept'][0, :]
1174 inParams[
'COMPQESYSSLOPE'][:] = parCat[
'compQeSysSlope'][0, :]
1175 inParams[
'PARFILTEROFFSET'][:] = parCat[
'parFilterOffset'][0, :]
1176 inParams[
'PARFILTEROFFSETFITFLAG'][:] = parCat[
'parFilterOffsetFitFlag'][0, :]
1177 inParams[
'PARRETRIEVEDLNPWVSCALE'] = parCat[
'parRetrievedLnPwvScale']
1178 inParams[
'PARRETRIEVEDLNPWVOFFSET'] = parCat[
'parRetrievedLnPwvOffset']
1179 inParams[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat[
'parRetrievedLnPwvNightlyOffset'][0, :]
1180 inParams[
'COMPABSTHROUGHPUT'][:] = parCat[
'compAbsThroughput'][0, :]
1181 inParams[
'COMPREFOFFSET'][:] = parCat[
'compRefOffset'][0, :]
1182 inParams[
'COMPREFSIGMA'][:] = parCat[
'compRefSigma'][0, :]
1183 inParams[
'COMPMIRRORCHROMATICITY'][:] = parCat[
'compMirrorChromaticity'][0, :]
1184 inParams[
'MIRRORCHROMATICITYPIVOT'][:] = parCat[
'mirrorChromaticityPivot'][0, :]
1185 inParams[
'COMPMEDIANSEDSLOPE'][:] = parCat[
'compMedianSedSlope'][0, :]
1186 inParams[
'COMPAPERCORRPIVOT'][:] = parCat[
'compAperCorrPivot'][0, :]
1187 inParams[
'COMPAPERCORRSLOPE'][:] = parCat[
'compAperCorrSlope'][0, :]
1188 inParams[
'COMPAPERCORRSLOPEERR'][:] = parCat[
'compAperCorrSlopeErr'][0, :]
1189 inParams[
'COMPAPERCORRRANGE'][:] = parCat[
'compAperCorrRange'][0, :]
1190 inParams[
'COMPMODELERREXPTIMEPIVOT'][:] = parCat[
'compModelErrExptimePivot'][0, :]
1191 inParams[
'COMPMODELERRFWHMPIVOT'][:] = parCat[
'compModelErrFwhmPivot'][0, :]
1192 inParams[
'COMPMODELERRSKYPIVOT'][:] = parCat[
'compModelErrSkyPivot'][0, :]
1193 inParams[
'COMPMODELERRPARS'][:] = parCat[
'compModelErrPars'][0, :]
1194 inParams[
'COMPEXPGRAY'][:] = parCat[
'compExpGray'][0, :]
1195 inParams[
'COMPVARGRAY'][:] = parCat[
'compVarGray'][0, :]
1196 inParams[
'COMPEXPDELTAMAGBKG'][:] = parCat[
'compExpDeltaMagBkg'][0, :]
1197 inParams[
'COMPNGOODSTARPEREXP'][:] = parCat[
'compNGoodStarPerExp'][0, :]
1198 inParams[
'COMPSIGFGCM'][:] = parCat[
'compSigFgcm'][0, :]
1199 inParams[
'COMPSIGMACAL'][:] = parCat[
'compSigmaCal'][0, :]
1200 inParams[
'COMPRETRIEVEDLNPWV'][:] = parCat[
'compRetrievedLnPwv'][0, :]
1201 inParams[
'COMPRETRIEVEDLNPWVRAW'][:] = parCat[
'compRetrievedLnPwvRaw'][0, :]
1202 inParams[
'COMPRETRIEVEDLNPWVFLAG'][:] = parCat[
'compRetrievedLnPwvFlag'][0, :]
1203 inParams[
'COMPRETRIEVEDTAUNIGHT'][:] = parCat[
'compRetrievedTauNight'][0, :]
1205 inSuperStar = np.zeros(parCat[
'superstarSize'][0, :], dtype=
'f8')
1206 inSuperStar[:, :, :, :] = parCat[
'superstar'][0, :].reshape(inSuperStar.shape)
1208 return (inParInfo, inParams, inSuperStar)
1210 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1212 Persist FGCM datasets through the butler.
1216 butler: `lsst.daf.persistence.Butler`
1217 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1218 Fgcm Fit cycle object
1222 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1224 parSchema = afwTable.Schema()
1227 lutFilterNameString = comma.join([n.decode(
'utf-8')
1228 for n
in parInfo[
'LUTFILTERNAMES'][0]])
1229 fitBandString = comma.join([n.decode(
'utf-8')
1230 for n
in parInfo[
'FITBANDS'][0]])
1232 parSchema = self.
_makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1233 lutFilterNameString, fitBandString)
1235 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1236 lutFilterNameString, fitBandString)
1238 butler.put(parCat,
'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1244 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1247 butler.put(flagStarCat,
'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1251 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
1252 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
1255 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1257 butler.put(zptCat,
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1262 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1264 butler.put(atmCat,
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1268 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1270 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
1272 butler.put(stdCat,
'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1274 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1275 lutFilterNameString, fitBandString):
1277 Make the parameter persistence schema
1281 parInfo: `numpy.ndarray`
1282 Parameter information returned by fgcm
1283 pars: `numpy.ndarray`
1284 Parameter values returned by fgcm
1285 parSuperStarFlat: `numpy.array`
1286 Superstar flat values returned by fgcm
1287 lutFilterNameString: `str`
1288 Combined string of all the lutFilterNames
1289 fitBandString: `str`
1290 Combined string of all the fitBands
1294 parSchema: `afwTable.schema`
1297 parSchema = afwTable.Schema()
1300 parSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
1301 parSchema.addField(
'lutFilterNames', type=str, doc=
'LUT Filter names in parameter file',
1302 size=len(lutFilterNameString))
1303 parSchema.addField(
'fitBands', type=str, doc=
'Bands that were fit',
1304 size=len(fitBandString))
1305 parSchema.addField(
'lnTauUnit', type=np.float64, doc=
'Step units for ln(AOD)')
1306 parSchema.addField(
'lnTauSlopeUnit', type=np.float64,
1307 doc=
'Step units for ln(AOD) slope')
1308 parSchema.addField(
'alphaUnit', type=np.float64, doc=
'Step units for alpha')
1309 parSchema.addField(
'lnPwvUnit', type=np.float64, doc=
'Step units for ln(pwv)')
1310 parSchema.addField(
'lnPwvSlopeUnit', type=np.float64,
1311 doc=
'Step units for ln(pwv) slope')
1312 parSchema.addField(
'lnPwvQuadraticUnit', type=np.float64,
1313 doc=
'Step units for ln(pwv) quadratic term')
1314 parSchema.addField(
'lnPwvGlobalUnit', type=np.float64,
1315 doc=
'Step units for global ln(pwv) parameters')
1316 parSchema.addField(
'o3Unit', type=np.float64, doc=
'Step units for O3')
1317 parSchema.addField(
'qeSysUnit', type=np.float64, doc=
'Step units for mirror gray')
1318 parSchema.addField(
'filterOffsetUnit', type=np.float64, doc=
'Step units for filter offset')
1319 parSchema.addField(
'hasExternalPwv', type=np.int32, doc=
'Parameters fit using external pwv')
1320 parSchema.addField(
'hasExternalTau', type=np.int32, doc=
'Parameters fit using external tau')
1323 parSchema.addField(
'parAlpha', type=
'ArrayD', doc=
'Alpha parameter vector',
1324 size=pars[
'PARALPHA'].size)
1325 parSchema.addField(
'parO3', type=
'ArrayD', doc=
'O3 parameter vector',
1326 size=pars[
'PARO3'].size)
1327 parSchema.addField(
'parLnTauIntercept', type=
'ArrayD',
1328 doc=
'ln(Tau) intercept parameter vector',
1329 size=pars[
'PARLNTAUINTERCEPT'].size)
1330 parSchema.addField(
'parLnTauSlope', type=
'ArrayD',
1331 doc=
'ln(Tau) slope parameter vector',
1332 size=pars[
'PARLNTAUSLOPE'].size)
1333 parSchema.addField(
'parLnPwvIntercept', type=
'ArrayD', doc=
'ln(pwv) intercept parameter vector',
1334 size=pars[
'PARLNPWVINTERCEPT'].size)
1335 parSchema.addField(
'parLnPwvSlope', type=
'ArrayD', doc=
'ln(pwv) slope parameter vector',
1336 size=pars[
'PARLNPWVSLOPE'].size)
1337 parSchema.addField(
'parLnPwvQuadratic', type=
'ArrayD', doc=
'ln(pwv) quadratic parameter vector',
1338 size=pars[
'PARLNPWVQUADRATIC'].size)
1339 parSchema.addField(
'parQeSysIntercept', type=
'ArrayD', doc=
'Mirror gray intercept parameter vector',
1340 size=pars[
'PARQESYSINTERCEPT'].size)
1341 parSchema.addField(
'compQeSysSlope', type=
'ArrayD', doc=
'Mirror gray slope parameter vector',
1342 size=pars[0][
'COMPQESYSSLOPE'].size)
1343 parSchema.addField(
'parFilterOffset', type=
'ArrayD', doc=
'Filter offset parameter vector',
1344 size=pars[
'PARFILTEROFFSET'].size)
1345 parSchema.addField(
'parFilterOffsetFitFlag', type=
'ArrayI', doc=
'Filter offset parameter fit flag',
1346 size=pars[
'PARFILTEROFFSETFITFLAG'].size)
1347 parSchema.addField(
'parRetrievedLnPwvScale', type=np.float64,
1348 doc=
'Global scale for retrieved ln(pwv)')
1349 parSchema.addField(
'parRetrievedLnPwvOffset', type=np.float64,
1350 doc=
'Global offset for retrieved ln(pwv)')
1351 parSchema.addField(
'parRetrievedLnPwvNightlyOffset', type=
'ArrayD',
1352 doc=
'Nightly offset for retrieved ln(pwv)',
1353 size=pars[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1354 parSchema.addField(
'compAbsThroughput', type=
'ArrayD',
1355 doc=
'Absolute throughput (relative to transmission curves)',
1356 size=pars[
'COMPABSTHROUGHPUT'].size)
1357 parSchema.addField(
'compRefOffset', type=
'ArrayD',
1358 doc=
'Offset between reference stars and calibrated stars',
1359 size=pars[
'COMPREFOFFSET'].size)
1360 parSchema.addField(
'compRefSigma', type=
'ArrayD',
1361 doc=
'Width of reference star/calibrated star distribution',
1362 size=pars[
'COMPREFSIGMA'].size)
1363 parSchema.addField(
'compMirrorChromaticity', type=
'ArrayD',
1364 doc=
'Computed mirror chromaticity terms',
1365 size=pars[
'COMPMIRRORCHROMATICITY'].size)
1366 parSchema.addField(
'mirrorChromaticityPivot', type=
'ArrayD',
1367 doc=
'Mirror chromaticity pivot mjd',
1368 size=pars[
'MIRRORCHROMATICITYPIVOT'].size)
1369 parSchema.addField(
'compMedianSedSlope', type=
'ArrayD',
1370 doc=
'Computed median SED slope (per band)',
1371 size=pars[
'COMPMEDIANSEDSLOPE'].size)
1372 parSchema.addField(
'compAperCorrPivot', type=
'ArrayD', doc=
'Aperture correction pivot',
1373 size=pars[
'COMPAPERCORRPIVOT'].size)
1374 parSchema.addField(
'compAperCorrSlope', type=
'ArrayD', doc=
'Aperture correction slope',
1375 size=pars[
'COMPAPERCORRSLOPE'].size)
1376 parSchema.addField(
'compAperCorrSlopeErr', type=
'ArrayD', doc=
'Aperture correction slope error',
1377 size=pars[
'COMPAPERCORRSLOPEERR'].size)
1378 parSchema.addField(
'compAperCorrRange', type=
'ArrayD', doc=
'Aperture correction range',
1379 size=pars[
'COMPAPERCORRRANGE'].size)
1380 parSchema.addField(
'compModelErrExptimePivot', type=
'ArrayD', doc=
'Model error exptime pivot',
1381 size=pars[
'COMPMODELERREXPTIMEPIVOT'].size)
1382 parSchema.addField(
'compModelErrFwhmPivot', type=
'ArrayD', doc=
'Model error fwhm pivot',
1383 size=pars[
'COMPMODELERRFWHMPIVOT'].size)
1384 parSchema.addField(
'compModelErrSkyPivot', type=
'ArrayD', doc=
'Model error sky pivot',
1385 size=pars[
'COMPMODELERRSKYPIVOT'].size)
1386 parSchema.addField(
'compModelErrPars', type=
'ArrayD', doc=
'Model error parameters',
1387 size=pars[
'COMPMODELERRPARS'].size)
1388 parSchema.addField(
'compExpGray', type=
'ArrayD', doc=
'Computed exposure gray',
1389 size=pars[
'COMPEXPGRAY'].size)
1390 parSchema.addField(
'compVarGray', type=
'ArrayD', doc=
'Computed exposure variance',
1391 size=pars[
'COMPVARGRAY'].size)
1392 parSchema.addField(
'compExpDeltaMagBkg', type=
'ArrayD',
1393 doc=
'Computed exposure offset due to background',
1394 size=pars[
'COMPEXPDELTAMAGBKG'].size)
1395 parSchema.addField(
'compNGoodStarPerExp', type=
'ArrayI',
1396 doc=
'Computed number of good stars per exposure',
1397 size=pars[
'COMPNGOODSTARPEREXP'].size)
1398 parSchema.addField(
'compSigFgcm', type=
'ArrayD', doc=
'Computed sigma_fgcm (intrinsic repeatability)',
1399 size=pars[
'COMPSIGFGCM'].size)
1400 parSchema.addField(
'compSigmaCal', type=
'ArrayD', doc=
'Computed sigma_cal (systematic error floor)',
1401 size=pars[
'COMPSIGMACAL'].size)
1402 parSchema.addField(
'compRetrievedLnPwv', type=
'ArrayD', doc=
'Retrieved ln(pwv) (smoothed)',
1403 size=pars[
'COMPRETRIEVEDLNPWV'].size)
1404 parSchema.addField(
'compRetrievedLnPwvRaw', type=
'ArrayD', doc=
'Retrieved ln(pwv) (raw)',
1405 size=pars[
'COMPRETRIEVEDLNPWVRAW'].size)
1406 parSchema.addField(
'compRetrievedLnPwvFlag', type=
'ArrayI', doc=
'Retrieved ln(pwv) Flag',
1407 size=pars[
'COMPRETRIEVEDLNPWVFLAG'].size)
1408 parSchema.addField(
'compRetrievedTauNight', type=
'ArrayD', doc=
'Retrieved tau (per night)',
1409 size=pars[
'COMPRETRIEVEDTAUNIGHT'].size)
1411 parSchema.addField(
'superstarSize', type=
'ArrayI', doc=
'Superstar matrix size',
1413 parSchema.addField(
'superstar', type=
'ArrayD', doc=
'Superstar matrix (flattened)',
1414 size=parSuperStarFlat.size)
1418 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1419 lutFilterNameString, fitBandString):
1421 Make the FGCM parameter catalog for persistence
1425 parSchema: `lsst.afw.table.Schema`
1426 Parameter catalog schema
1427 pars: `numpy.ndarray`
1428 FGCM parameters to put into parCat
1429 parSuperStarFlat: `numpy.array`
1430 FGCM superstar flat array to put into parCat
1431 lutFilterNameString: `str`
1432 Combined string of all the lutFilterNames
1433 fitBandString: `str`
1434 Combined string of all the fitBands
1438 parCat: `afwTable.BasicCatalog`
1439 Atmosphere and instrumental model parameter catalog for persistence
1442 parCat = afwTable.BaseCatalog(parSchema)
1447 rec = parCat.addNew()
1450 rec[
'nCcd'] = parInfo[
'NCCD']
1451 rec[
'lutFilterNames'] = lutFilterNameString
1452 rec[
'fitBands'] = fitBandString
1454 rec[
'hasExternalPwv'] = 0
1455 rec[
'hasExternalTau'] = 0
1459 scalarNames = [
'parRetrievedLnPwvScale',
'parRetrievedLnPwvOffset']
1461 arrNames = [
'parAlpha',
'parO3',
'parLnTauIntercept',
'parLnTauSlope',
1462 'parLnPwvIntercept',
'parLnPwvSlope',
'parLnPwvQuadratic',
1463 'parQeSysIntercept',
'compQeSysSlope',
1464 'parRetrievedLnPwvNightlyOffset',
'compAperCorrPivot',
1465 'parFilterOffset',
'parFilterOffsetFitFlag',
1466 'compAbsThroughput',
'compRefOffset',
'compRefSigma',
1467 'compMirrorChromaticity',
'mirrorChromaticityPivot',
1468 'compAperCorrSlope',
'compAperCorrSlopeErr',
'compAperCorrRange',
1469 'compModelErrExptimePivot',
'compModelErrFwhmPivot',
1470 'compModelErrSkyPivot',
'compModelErrPars',
1471 'compExpGray',
'compVarGray',
'compNGoodStarPerExp',
'compSigFgcm',
1472 'compSigmaCal',
'compExpDeltaMagBkg',
'compMedianSedSlope',
1473 'compRetrievedLnPwv',
'compRetrievedLnPwvRaw',
'compRetrievedLnPwvFlag',
1474 'compRetrievedTauNight']
1476 for scalarName
in scalarNames:
1477 rec[scalarName] = pars[scalarName.upper()]
1479 for arrName
in arrNames:
1480 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1483 rec[
'superstarSize'][:] = parSuperStarFlat.shape
1484 rec[
'superstar'][:] = parSuperStarFlat.flatten()
1488 def _makeFlagStarSchema(self):
1490 Make the flagged-stars schema
1494 flagStarSchema: `lsst.afw.table.Schema`
1497 flagStarSchema = afwTable.Schema()
1499 flagStarSchema.addField(
'objId', type=np.int32, doc=
'FGCM object id')
1500 flagStarSchema.addField(
'objFlag', type=np.int32, doc=
'FGCM object flag')
1502 return flagStarSchema
1504 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1506 Make the flagged star catalog for persistence
1510 flagStarSchema: `lsst.afw.table.Schema`
1512 flagStarStruct: `numpy.ndarray`
1513 Flagged star structure from fgcm
1517 flagStarCat: `lsst.afw.table.BaseCatalog`
1518 Flagged star catalog for persistence
1521 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1522 flagStarCat.resize(flagStarStruct.size)
1524 flagStarCat[
'objId'][:] = flagStarStruct[
'OBJID']
1525 flagStarCat[
'objFlag'][:] = flagStarStruct[
'OBJFLAG']