23 """Perform a single fit cycle of FGCM. 25 This task runs a single "fit cycle" of fgcm. Prior to running this task 26 one must run both fgcmMakeLut (to construct the atmosphere and instrumental 27 look-up-table) and fgcmBuildStars (to extract visits and star observations 30 The fgcmFitCycle is meant to be run multiple times, and is tracked by the 31 'cycleNumber'. After each run of the fit cycle, diagnostic plots should 32 be inspected to set parameters for outlier rejection on the following 33 cycle. Please see the fgcmcal Cookbook for details. 42 import lsst.pex.config
as pexConfig
43 import lsst.pipe.base
as pipeBase
44 import lsst.afw.table
as afwTable
46 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
47 from .utilities
import extractReferenceMags
48 from .utilities
import computeCcdOffsets, makeZptSchema, makeZptCat
49 from .utilities
import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50 from .sedterms
import SedboundarytermDict, SedtermDict
54 __all__ = [
'FgcmFitCycleConfig',
'FgcmFitCycleTask',
'FgcmFitCycleRunner']
58 """Config for FgcmFitCycle""" 60 bands = pexConfig.ListField(
61 doc=
"Bands to run calibration",
65 fitFlag = pexConfig.ListField(
66 doc=(
"Flag for which bands are directly constrained in the FGCM fit. " 67 "Bands set to 0 will have the atmosphere constrained from observations " 68 "in other bands on the same night. Must be same length as config.bands, " 69 "and matched band-by-band."),
73 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 74 "It will be removed after v20. Use fitBands instead."),
76 fitBands = pexConfig.ListField(
77 doc=(
"Bands to use in atmospheric fit. The bands not listed here will have " 78 "the atmosphere constrained from the 'fitBands' on the same night. " 79 "Must be a subset of `config.bands`"),
83 requiredFlag = pexConfig.ListField(
84 doc=(
"Flag for which bands are required for a star to be considered a calibration " 85 "star in the FGCM fit. Typically this should be the same as fitFlag. Must " 86 "be same length as config.bands, and matched band-by-band."),
90 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 91 "It will be removed after v20. Use requiredBands instead."),
93 requiredBands = pexConfig.ListField(
94 doc=(
"Bands that are required for a star to be considered a calibration star. " 95 "Must be a subset of `config.bands`"),
99 filterMap = pexConfig.DictField(
100 doc=
"Mapping from 'filterName' to band.",
105 doReferenceCalibration = pexConfig.Field(
106 doc=
"Use reference catalog as additional constraint on calibration",
110 refStarSnMin = pexConfig.Field(
111 doc=
"Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
115 refStarOutlierNSig = pexConfig.Field(
116 doc=(
"Number of sigma compared to average mag for reference star to be considered an outlier. " 117 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
121 applyRefStarColorCuts = pexConfig.Field(
122 doc=
"Apply color cuts to reference stars?",
126 nCore = pexConfig.Field(
127 doc=
"Number of cores to use",
131 nStarPerRun = pexConfig.Field(
132 doc=
"Number of stars to run in each chunk",
136 nExpPerRun = pexConfig.Field(
137 doc=
"Number of exposures to run in each chunk",
141 reserveFraction = pexConfig.Field(
142 doc=
"Fraction of stars to reserve for testing",
146 freezeStdAtmosphere = pexConfig.Field(
147 doc=
"Freeze atmosphere parameters to standard (for testing)",
151 precomputeSuperStarInitialCycle = pexConfig.Field(
152 doc=
"Precompute superstar flat for initial cycle",
156 superStarSubCcd = pexConfig.Field(
157 doc=
"Compute superstar flat on sub-ccd scale",
161 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 162 "It will be removed after v20. Use superStarSubCcdDict instead."),
164 superStarSubCcdDict = pexConfig.DictField(
165 doc=(
"Per-band specification on whether to compute superstar flat on sub-ccd scale. " 166 "Must have one entry per band."),
171 superStarSubCcdChebyshevOrder = pexConfig.Field(
172 doc=(
"Order of the 2D chebyshev polynomials for sub-ccd superstar fit. " 173 "Global default is first-order polynomials, and should be overridden " 174 "on a camera-by-camera basis depending on the ISR."),
178 superStarSubCcdTriangular = pexConfig.Field(
179 doc=(
"Should the sub-ccd superstar chebyshev matrix be triangular to " 180 "suppress high-order cross terms?"),
184 superStarSigmaClip = pexConfig.Field(
185 doc=
"Number of sigma to clip outliers when selecting for superstar flats",
189 ccdGraySubCcd = pexConfig.Field(
190 doc=
"Compute CCD gray terms on sub-ccd scale",
194 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 195 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
197 ccdGraySubCcdDict = pexConfig.DictField(
198 doc=(
"Per-band specification on whether to compute achromatic per-ccd residual " 199 "('ccd gray') on a sub-ccd scale."),
204 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
205 doc=
"Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
209 ccdGraySubCcdTriangular = pexConfig.Field(
210 doc=(
"Should the sub-ccd gray chebyshev matrix be triangular to " 211 "suppress high-order cross terms?"),
215 cycleNumber = pexConfig.Field(
216 doc=(
"FGCM fit cycle number. This is automatically incremented after each run " 217 "and stage of outlier rejection. See cookbook for details."),
221 isFinalCycle = pexConfig.Field(
222 doc=(
"Is this the final cycle of the fitting? Will automatically compute final " 223 "selection of stars and photometric exposures, and will output zeropoints " 224 "and standard stars for use in fgcmOutputProducts"),
228 maxIterBeforeFinalCycle = pexConfig.Field(
229 doc=(
"Maximum fit iterations, prior to final cycle. The number of iterations " 230 "will always be 0 in the final cycle for cleanup and final selection."),
234 utBoundary = pexConfig.Field(
235 doc=
"Boundary (in UTC) from day-to-day",
239 washMjds = pexConfig.ListField(
240 doc=
"Mirror wash MJDs",
244 epochMjds = pexConfig.ListField(
245 doc=
"Epoch boundaries in MJD",
249 minObsPerBand = pexConfig.Field(
250 doc=
"Minimum good observations per band",
256 latitude = pexConfig.Field(
257 doc=
"Observatory latitude",
261 pixelScale = pexConfig.Field(
262 doc=
"Pixel scale (arcsec/pixel) (temporary)",
264 deprecated=(
"This field is no longer used, and has been deprecated by DM-16490. " 265 "It will be removed after v19."),
268 brightObsGrayMax = pexConfig.Field(
269 doc=
"Maximum gray extinction to be considered bright observation",
273 minStarPerCcd = pexConfig.Field(
274 doc=(
"Minimum number of good stars per CCD to be used in calibration fit. " 275 "CCDs with fewer stars will have their calibration estimated from other " 276 "CCDs in the same visit, with zeropoint error increased accordingly."),
280 minCcdPerExp = pexConfig.Field(
281 doc=(
"Minimum number of good CCDs per exposure/visit to be used in calibration fit. " 282 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
286 maxCcdGrayErr = pexConfig.Field(
287 doc=
"Maximum error on CCD gray offset to be considered photometric",
291 minStarPerExp = pexConfig.Field(
292 doc=(
"Minimum number of good stars per exposure/visit to be used in calibration fit. " 293 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
297 minExpPerNight = pexConfig.Field(
298 doc=
"Minimum number of good exposures/visits to consider a partly photometric night",
302 expGrayInitialCut = pexConfig.Field(
303 doc=(
"Maximum exposure/visit gray value for initial selection of possible photometric " 308 expGrayPhotometricCut = pexConfig.ListField(
309 doc=(
"Maximum (negative) exposure gray for a visit to be considered photometric. " 310 "Must be same length as config.bands, and matched band-by-band."),
314 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 315 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
317 expGrayPhotometricCutDict = pexConfig.DictField(
318 doc=(
"Per-band specification on maximum (negative) achromatic exposure residual " 319 "('gray term') for a visit to be considered photometric. Must have one " 320 "entry per band. Broad-band filters should be -0.05."),
325 expGrayHighCut = pexConfig.ListField(
326 doc=(
"Maximum (positive) exposure gray for a visit to be considered photometric. " 327 "Must be same length as config.bands, and matched band-by-band."),
331 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 332 "It will be removed after v20. Use expGrayHighCutDict instead."),
334 expGrayHighCutDict = pexConfig.DictField(
335 doc=(
"Per-band specification on maximum (positive) achromatic exposure residual " 336 "('gray term') for a visit to be considered photometric. Must have one " 337 "entry per band. Broad-band filters should be 0.2."),
342 expGrayRecoverCut = pexConfig.Field(
343 doc=(
"Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. " 344 "Visits with more gray extinction will only get CCD zeropoints if there are " 345 "sufficient star observations (minStarPerCcd) on that CCD."),
349 expVarGrayPhotometricCut = pexConfig.Field(
350 doc=
"Maximum exposure variance to be considered possibly photometric",
354 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 355 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
357 expVarGrayPhotometricCutDict = pexConfig.DictField(
358 doc=(
"Per-band specification on maximum exposure variance to be considered possibly " 359 "photometric. Must have one entry per band. Broad-band filters should be " 365 expGrayErrRecoverCut = pexConfig.Field(
366 doc=(
"Maximum exposure gray error to be able to recover bad ccds via interpolation. " 367 "Visits with more gray variance will only get CCD zeropoints if there are " 368 "sufficient star observations (minStarPerCcd) on that CCD."),
372 aperCorrFitNBins = pexConfig.Field(
373 doc=(
"Number of aperture bins used in aperture correction fit. When set to 0" 374 "no fit will be performed, and the config.aperCorrInputSlopes will be " 375 "used if available."),
379 aperCorrInputSlopes = pexConfig.ListField(
380 doc=(
"Aperture correction input slope parameters. These are used on the first " 381 "fit iteration, and aperture correction parameters will be updated from " 382 "the data if config.aperCorrFitNBins > 0. It is recommended to set this" 383 "when there is insufficient data to fit the parameters (e.g. tract mode). " 384 "If set, must be same length as config.bands, and matched band-by-band."),
388 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 389 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
391 aperCorrInputSlopeDict = pexConfig.DictField(
392 doc=(
"Per-band specification of aperture correction input slope parameters. These " 393 "are used on the first fit iteration, and aperture correction parameters will " 394 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended " 395 "to set this when there is insufficient data to fit the parameters (e.g. " 401 sedFudgeFactors = pexConfig.ListField(
402 doc=(
"Fudge factors for computing linear SED from colors. Must be same length as " 403 "config.bands, and matched band-by-band."),
407 deprecated=(
"This field has been deprecated and will be removed after v20. " 408 "Please use sedSlopeTermMap and sedSlopeMap."),
410 sedboundaryterms = pexConfig.ConfigField(
411 doc=
"Mapping from bands to SED boundary term names used is sedterms.",
412 dtype=SedboundarytermDict,
414 sedterms = pexConfig.ConfigField(
415 doc=
"Mapping from terms to bands for fgcm linear SED approximations.",
418 sigFgcmMaxErr = pexConfig.Field(
419 doc=
"Maximum mag error for fitting sigma_FGCM",
423 sigFgcmMaxEGray = pexConfig.ListField(
424 doc=(
"Maximum (absolute) gray value for observation in sigma_FGCM. " 425 "May be 1 element (same for all bands) or the same length as config.bands."),
429 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 430 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
432 sigFgcmMaxEGrayDict = pexConfig.DictField(
433 doc=(
"Per-band specification for maximum (absolute) achromatic residual (gray value) " 434 "for observations in sigma_fgcm (raw repeatability). Broad-band filters " 440 ccdGrayMaxStarErr = pexConfig.Field(
441 doc=(
"Maximum error on a star observation to use in ccd gray (achromatic residual) " 446 approxThroughput = pexConfig.ListField(
447 doc=(
"Approximate overall throughput at start of calibration observations. " 448 "May be 1 element (same for all bands) or the same length as config.bands."),
452 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 453 "It will be removed after v20. Use approxThroughputDict instead."),
455 approxThroughputDict = pexConfig.DictField(
456 doc=(
"Per-band specification of the approximate overall throughput at the start of " 457 "calibration observations. Must have one entry per band. Typically should " 463 sigmaCalRange = pexConfig.ListField(
464 doc=
"Allowed range for systematic error floor estimation",
466 default=(0.001, 0.003),
468 sigmaCalFitPercentile = pexConfig.ListField(
469 doc=
"Magnitude percentile range to fit systematic error floor",
471 default=(0.05, 0.15),
473 sigmaCalPlotPercentile = pexConfig.ListField(
474 doc=
"Magnitude percentile range to plot systematic error floor",
476 default=(0.05, 0.95),
478 sigma0Phot = pexConfig.Field(
479 doc=
"Systematic error floor for all zeropoints",
483 mapLongitudeRef = pexConfig.Field(
484 doc=
"Reference longitude for plotting maps",
488 mapNSide = pexConfig.Field(
489 doc=
"Healpix nside for plotting maps",
493 outfileBase = pexConfig.Field(
494 doc=
"Filename start for plot output files",
498 starColorCuts = pexConfig.ListField(
499 doc=
"Encoded star-color cuts (to be cleaned up)",
501 default=(
"NO_DATA",),
503 colorSplitIndices = pexConfig.ListField(
504 doc=
"Band indices to use to split stars by color",
508 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 509 "It will be removed after v20. Use colorSplitBands instead."),
511 colorSplitBands = pexConfig.ListField(
512 doc=
"Band names to use to split stars by color. Must have 2 entries.",
517 modelMagErrors = pexConfig.Field(
518 doc=
"Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
522 useQuadraticPwv = pexConfig.Field(
523 doc=
"Model PWV with a quadratic term for variation through the night?",
527 instrumentParsPerBand = pexConfig.Field(
528 doc=(
"Model instrumental parameters per band? " 529 "Otherwise, instrumental parameters (QE changes with time) are " 530 "shared among all bands."),
534 instrumentSlopeMinDeltaT = pexConfig.Field(
535 doc=(
"Minimum time change (in days) between observations to use in constraining " 536 "instrument slope."),
540 fitMirrorChromaticity = pexConfig.Field(
541 doc=
"Fit (intraband) mirror chromatic term?",
545 coatingMjds = pexConfig.ListField(
546 doc=
"Mirror coating dates in MJD",
550 outputStandardsBeforeFinalCycle = pexConfig.Field(
551 doc=
"Output standard stars prior to final cycle? Used in debugging.",
555 outputZeropointsBeforeFinalCycle = pexConfig.Field(
556 doc=
"Output standard stars prior to final cycle? Used in debugging.",
560 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
561 doc=(
"Use star repeatability (instead of exposures) for computing photometric " 562 "cuts? Recommended for tract mode or bands with few exposures. " 563 "May be 1 element (same for all bands) or the same length as config.bands."),
567 deprecated=(
"This field is no longer used, and has been deprecated by DM-23699. " 568 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
570 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
571 doc=(
"Per-band specification on whether to use star repeatability (instead of exposures) " 572 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
577 autoPhotometricCutNSig = pexConfig.Field(
578 doc=(
"Number of sigma for automatic computation of (low) photometric cut. " 579 "Cut is based on exposure gray width (per band), unless " 580 "useRepeatabilityForExpGrayCuts is set, in which case the star " 581 "repeatability is used (also per band)."),
585 autoHighCutNSig = pexConfig.Field(
586 doc=(
"Number of sigma for automatic computation of (high) outlier cut. " 587 "Cut is based on exposure gray width (per band), unless " 588 "useRepeatabilityForExpGrayCuts is set, in which case the star " 589 "repeatability is used (also per band)."),
593 quietMode = pexConfig.Field(
594 doc=
"Be less verbose with logging.",
605 for band
in self.fitBands:
606 if band
not in self.bands:
607 msg =
'fitBand %s not in bands' % (band)
608 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
609 for band
in self.requiredBands:
610 if band
not in self.bands:
611 msg =
'requiredBand %s not in bands' % (band)
612 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
613 for band
in self.colorSplitBands:
614 if band
not in self.bands:
615 msg =
'colorSplitBand %s not in bands' % (band)
616 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
617 for band
in self.bands:
618 if band
not in self.superStarSubCcdDict:
619 msg =
'band %s not in superStarSubCcdDict' % (band)
620 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
622 if band
not in self.ccdGraySubCcdDict:
623 msg =
'band %s not in ccdGraySubCcdDict' % (band)
624 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
626 if band
not in self.expGrayPhotometricCutDict:
627 msg =
'band %s not in expGrayPhotometricCutDict' % (band)
628 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
630 if band
not in self.expGrayHighCutDict:
631 msg =
'band %s not in expGrayHighCutDict' % (band)
632 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
634 if band
not in self.expVarGrayPhotometricCutDict:
635 msg =
'band %s not in expVarGrayPhotometricCutDict' % (band)
636 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
638 if band
not in self.sigFgcmMaxEGrayDict:
639 msg =
'band %s not in sigFgcmMaxEGrayDict' % (band)
640 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
642 if band
not in self.approxThroughputDict:
643 msg =
'band %s not in approxThroughputDict' % (band)
644 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
646 if band
not in self.useRepeatabilityForExpGrayCutsDict:
647 msg =
'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
648 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
653 """Subclass of TaskRunner for fgcmFitCycleTask 655 fgcmFitCycleTask.run() takes one argument, the butler, and uses 656 stars and visits previously extracted from dataRefs by 658 This Runner does not perform any dataRef parallelization, but the FGCM 659 code called by the Task uses python multiprocessing (see the "ncores" 666 Return a list with one element, the butler. 668 return [parsedCmd.butler]
674 butler: `lsst.daf.persistence.Butler` 678 exitStatus: `list` with `pipeBase.Struct` 679 exitStatus (0: success; 1: failure) 682 task = self.TaskClass(config=self.config, log=self.log)
686 task.runDataRef(butler)
689 task.runDataRef(butler)
690 except Exception
as e:
692 task.log.fatal(
"Failed: %s" % e)
693 if not isinstance(e, pipeBase.TaskError):
694 traceback.print_exc(file=sys.stderr)
696 task.writeMetadata(butler)
699 return [pipeBase.Struct(exitStatus=exitStatus)]
703 Run the task, with no multiprocessing 707 parsedCmd: ArgumentParser parsed command line 712 if self.precall(parsedCmd):
715 resultList = self(targetList[0])
722 Run Single fit cycle for FGCM global calibration 725 ConfigClass = FgcmFitCycleConfig
726 RunnerClass = FgcmFitCycleRunner
727 _DefaultName =
"fgcmFitCycle" 731 Instantiate an fgcmFitCycle. 735 butler : `lsst.daf.persistence.Butler` 738 pipeBase.CmdLineTask.__init__(self, **kwargs)
741 def _getMetadataName(self):
747 Run a single fit cycle for FGCM 751 butler: `lsst.daf.persistence.Butler` 757 """Write the configuration used for processing the data, or check that an existing 758 one is equal to the new one if present. This is an override of the regular 759 version from pipe_base that knows about fgcmcycle. 763 butler : `lsst.daf.persistence.Butler` 764 Data butler used to write the config. The config is written to dataset type 765 `CmdLineTask._getConfigName`. 766 clobber : `bool`, optional 767 A boolean flag that controls what happens if a config already has been saved: 768 - `True`: overwrite or rename the existing config, depending on ``doBackup``. 769 - `False`: raise `TaskError` if this config does not match the existing config. 770 doBackup : `bool`, optional 771 Set to `True` to backup the config files if clobbering. 773 configName = self._getConfigName()
774 if configName
is None:
777 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
778 elif butler.datasetExists(configName, write=
True, fgcmcycle=self.config.cycleNumber):
781 oldConfig = butler.get(configName, immediate=
True, fgcmcycle=self.config.cycleNumber)
782 except Exception
as exc:
783 raise type(exc)(
"Unable to read stored config file %s (%s); consider using --clobber-config" %
786 def logConfigMismatch(msg):
787 self.log.fatal(
"Comparing configuration: %s", msg)
789 if not self.config.compare(oldConfig, shortcut=
False, output=logConfigMismatch):
790 raise pipeBase.TaskError(
791 (
"Config does not match existing task config %r on disk; tasks configurations " +
792 "must be consistent within the same output repo (override with --clobber-config)") %
795 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
797 def _fgcmFitCycle(self, butler):
803 butler: `lsst.daf.persistence.Butler` 809 self.
maxIter = self.config.maxIterBeforeFinalCycle
814 if self.config.isFinalCycle:
823 camera = butler.get(
'camera')
828 lutCat = butler.get(
'fgcmLookUpTable')
829 fgcmLut, lutIndexVals, lutStd =
translateFgcmLut(lutCat, dict(self.config.filterMap))
835 visitCat = butler.get(
'fgcmVisitCatalog')
843 noFitsDict = {
'lutIndex': lutIndexVals,
845 'expInfo': fgcmExpInfo,
846 'ccdOffsets': ccdOffsets}
849 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
850 noFitsDict=noFitsDict, noOutput=
True)
853 if (fgcmFitCycle.initialCycle):
855 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
860 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
866 lastCycle = configDict[
'cycleNumber'] - 1
869 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
871 starObs = butler.get(
'fgcmStarObservations')
872 starIds = butler.get(
'fgcmStarIds')
873 starIndices = butler.get(
'fgcmStarIndices')
876 if butler.datasetExists(
'fgcmFlaggedStars', fgcmcycle=lastCycle):
877 flaggedStars = butler.get(
'fgcmFlaggedStars', fgcmcycle=lastCycle)
878 flagId = flaggedStars[
'objId'][:]
879 flagFlag = flaggedStars[
'objFlag'][:]
884 if self.config.doReferenceCalibration:
885 refStars = butler.get(
'fgcmReferenceStars')
889 self.config.filterMap)
890 refId = refStars[
'fgcm_id'][:]
899 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'], starObs[
'visit'][starIndices[
'obsIndex']])
911 conv = starObs[0][
'ra'].asDegrees() / float(starObs[0][
'ra'])
913 fgcmStars.loadStars(fgcmPars,
914 starObs[
'visit'][starIndices[
'obsIndex']],
915 starObs[
'ccd'][starIndices[
'obsIndex']],
916 starObs[
'ra'][starIndices[
'obsIndex']] * conv,
917 starObs[
'dec'][starIndices[
'obsIndex']] * conv,
918 starObs[
'instMag'][starIndices[
'obsIndex']],
919 starObs[
'instMagErr'][starIndices[
'obsIndex']],
920 fgcmExpInfo[
'FILTERNAME'][visitIndex],
921 starIds[
'fgcm_id'][:],
924 starIds[
'obsArrIndex'][:],
926 obsX=starObs[
'x'][starIndices[
'obsIndex']],
927 obsY=starObs[
'y'][starIndices[
'obsIndex']],
928 psfCandidate=starObs[
'psf_candidate'][starIndices[
'obsIndex']],
946 fgcmFitCycle.setLUT(fgcmLut)
947 fgcmFitCycle.setStars(fgcmStars)
948 fgcmFitCycle.setPars(fgcmPars)
951 fgcmFitCycle.finishSetup()
965 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i])
for 966 i, b
in enumerate(self.config.bands)}
967 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i])
for 968 i, band
in enumerate(self.config.bands)}
970 outConfig = copy.copy(self.config)
971 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
972 precomputeSuperStarInitialCycle=
False,
973 freezeStdAtmosphere=
False,
974 expGrayPhotometricCutDict=updatedPhotometricCutDict,
975 expGrayHighCutDict=updatedHighCutDict)
976 configFileName =
'%s_cycle%02d_config.py' % (outConfig.outfileBase,
977 outConfig.cycleNumber)
978 outConfig.save(configFileName)
980 if self.config.isFinalCycle == 1:
982 self.log.info(
"Everything is in place to run fgcmOutputProducts.py")
984 self.log.info(
"Saved config for next cycle to %s" % (configFileName))
985 self.log.info(
"Be sure to look at:")
986 self.log.info(
" config.expGrayPhotometricCut")
987 self.log.info(
" config.expGrayHighCut")
988 self.log.info(
"If you are satisfied with the fit, please set:")
989 self.log.info(
" config.isFinalCycle = True")
991 def _checkDatasetsExist(self, butler):
993 Check if necessary datasets exist to run fgcmFitCycle 997 butler: `lsst.daf.persistence.Butler` 1002 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds, 1003 fgcmStarIndices, fgcmLookUpTable datasets do not exist. 1004 If cycleNumber > 0, then also checks for fgcmFitParameters, 1008 if not butler.datasetExists(
'fgcmVisitCatalog'):
1009 raise RuntimeError(
"Could not find fgcmVisitCatalog in repo!")
1010 if not butler.datasetExists(
'fgcmStarObservations'):
1011 raise RuntimeError(
"Could not find fgcmStarObservations in repo!")
1012 if not butler.datasetExists(
'fgcmStarIds'):
1013 raise RuntimeError(
"Could not find fgcmStarIds in repo!")
1014 if not butler.datasetExists(
'fgcmStarIndices'):
1015 raise RuntimeError(
"Could not find fgcmStarIndices in repo!")
1016 if not butler.datasetExists(
'fgcmLookUpTable'):
1017 raise RuntimeError(
"Could not find fgcmLookUpTable in repo!")
1020 if (self.config.cycleNumber > 0):
1021 if not butler.datasetExists(
'fgcmFitParameters',
1022 fgcmcycle=self.config.cycleNumber-1):
1023 raise RuntimeError(
"Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1024 (self.config.cycleNumber-1))
1025 if not butler.datasetExists(
'fgcmFlaggedStars',
1026 fgcmcycle=self.config.cycleNumber-1):
1027 raise RuntimeError(
"Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1028 (self.config.cycleNumber-1))
1031 if self.config.doReferenceCalibration:
1032 if not butler.datasetExists(
'fgcmReferenceStars'):
1033 raise RuntimeError(
"Could not find fgcmReferenceStars in repo, and " 1034 "doReferenceCalibration is True.")
1036 def _loadParameters(self, butler):
1038 Load FGCM parameters from a previous fit cycle 1042 butler: `lsst.daf.persistence.Butler` 1046 inParInfo: `numpy.ndarray` 1047 Numpy array parameter information formatted for input to fgcm 1048 inParameters: `numpy.ndarray` 1049 Numpy array parameter values formatted for input to fgcm 1050 inSuperStar: `numpy.array` 1051 Superstar flat formatted for input to fgcm 1055 parCat = butler.get(
'fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
1057 parLutFilterNames = np.array(parCat[0][
'lutFilterNames'].split(
','))
1058 parFitBands = np.array(parCat[0][
'fitBands'].split(
','))
1059 parNotFitBands = np.array(parCat[0][
'notFitBands'].split(
','))
1061 inParInfo = np.zeros(1, dtype=[(
'NCCD',
'i4'),
1062 (
'LUTFILTERNAMES', parLutFilterNames.dtype.str,
1063 (parLutFilterNames.size, )),
1064 (
'FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1065 (
'NOTFITBANDS', parNotFitBands.dtype.str, (parNotFitBands.size, )),
1066 (
'LNTAUUNIT',
'f8'),
1067 (
'LNTAUSLOPEUNIT',
'f8'),
1068 (
'ALPHAUNIT',
'f8'),
1069 (
'LNPWVUNIT',
'f8'),
1070 (
'LNPWVSLOPEUNIT',
'f8'),
1071 (
'LNPWVQUADRATICUNIT',
'f8'),
1072 (
'LNPWVGLOBALUNIT',
'f8'),
1074 (
'QESYSUNIT',
'f8'),
1075 (
'FILTEROFFSETUNIT',
'f8'),
1076 (
'HASEXTERNALPWV',
'i2'),
1077 (
'HASEXTERNALTAU',
'i2')])
1078 inParInfo[
'NCCD'] = parCat[
'nCcd']
1079 inParInfo[
'LUTFILTERNAMES'][:] = parLutFilterNames
1080 inParInfo[
'FITBANDS'][:] = parFitBands
1081 inParInfo[
'NOTFITBANDS'][:] = parNotFitBands
1082 inParInfo[
'HASEXTERNALPWV'] = parCat[
'hasExternalPwv']
1083 inParInfo[
'HASEXTERNALTAU'] = parCat[
'hasExternalTau']
1085 inParams = np.zeros(1, dtype=[(
'PARALPHA',
'f8', (parCat[
'parAlpha'].size, )),
1086 (
'PARO3',
'f8', (parCat[
'parO3'].size, )),
1087 (
'PARLNTAUINTERCEPT',
'f8',
1088 (parCat[
'parLnTauIntercept'].size, )),
1089 (
'PARLNTAUSLOPE',
'f8',
1090 (parCat[
'parLnTauSlope'].size, )),
1091 (
'PARLNPWVINTERCEPT',
'f8',
1092 (parCat[
'parLnPwvIntercept'].size, )),
1093 (
'PARLNPWVSLOPE',
'f8',
1094 (parCat[
'parLnPwvSlope'].size, )),
1095 (
'PARLNPWVQUADRATIC',
'f8',
1096 (parCat[
'parLnPwvQuadratic'].size, )),
1097 (
'PARQESYSINTERCEPT',
'f8',
1098 (parCat[
'parQeSysIntercept'].size, )),
1099 (
'COMPQESYSSLOPE',
'f8',
1100 (parCat[
'compQeSysSlope'].size, )),
1101 (
'PARFILTEROFFSET',
'f8',
1102 (parCat[
'parFilterOffset'].size, )),
1103 (
'PARFILTEROFFSETFITFLAG',
'i2',
1104 (parCat[
'parFilterOffsetFitFlag'].size, )),
1105 (
'PARRETRIEVEDLNPWVSCALE',
'f8'),
1106 (
'PARRETRIEVEDLNPWVOFFSET',
'f8'),
1107 (
'PARRETRIEVEDLNPWVNIGHTLYOFFSET',
'f8',
1108 (parCat[
'parRetrievedLnPwvNightlyOffset'].size, )),
1109 (
'COMPABSTHROUGHPUT',
'f8',
1110 (parCat[
'compAbsThroughput'].size, )),
1111 (
'COMPREFOFFSET',
'f8',
1112 (parCat[
'compRefOffset'].size, )),
1113 (
'COMPREFSIGMA',
'f8',
1114 (parCat[
'compRefSigma'].size, )),
1115 (
'COMPMIRRORCHROMATICITY',
'f8',
1116 (parCat[
'compMirrorChromaticity'].size, )),
1117 (
'MIRRORCHROMATICITYPIVOT',
'f8',
1118 (parCat[
'mirrorChromaticityPivot'].size, )),
1119 (
'COMPAPERCORRPIVOT',
'f8',
1120 (parCat[
'compAperCorrPivot'].size, )),
1121 (
'COMPAPERCORRSLOPE',
'f8',
1122 (parCat[
'compAperCorrSlope'].size, )),
1123 (
'COMPAPERCORRSLOPEERR',
'f8',
1124 (parCat[
'compAperCorrSlopeErr'].size, )),
1125 (
'COMPAPERCORRRANGE',
'f8',
1126 (parCat[
'compAperCorrRange'].size, )),
1127 (
'COMPMODELERREXPTIMEPIVOT',
'f8',
1128 (parCat[
'compModelErrExptimePivot'].size, )),
1129 (
'COMPMODELERRFWHMPIVOT',
'f8',
1130 (parCat[
'compModelErrFwhmPivot'].size, )),
1131 (
'COMPMODELERRSKYPIVOT',
'f8',
1132 (parCat[
'compModelErrSkyPivot'].size, )),
1133 (
'COMPMODELERRPARS',
'f8',
1134 (parCat[
'compModelErrPars'].size, )),
1135 (
'COMPEXPGRAY',
'f8',
1136 (parCat[
'compExpGray'].size, )),
1137 (
'COMPVARGRAY',
'f8',
1138 (parCat[
'compVarGray'].size, )),
1139 (
'COMPNGOODSTARPEREXP',
'i4',
1140 (parCat[
'compNGoodStarPerExp'].size, )),
1141 (
'COMPSIGFGCM',
'f8',
1142 (parCat[
'compSigFgcm'].size, )),
1143 (
'COMPSIGMACAL',
'f8',
1144 (parCat[
'compSigmaCal'].size, )),
1145 (
'COMPRETRIEVEDLNPWV',
'f8',
1146 (parCat[
'compRetrievedLnPwv'].size, )),
1147 (
'COMPRETRIEVEDLNPWVRAW',
'f8',
1148 (parCat[
'compRetrievedLnPwvRaw'].size, )),
1149 (
'COMPRETRIEVEDLNPWVFLAG',
'i2',
1150 (parCat[
'compRetrievedLnPwvFlag'].size, )),
1151 (
'COMPRETRIEVEDTAUNIGHT',
'f8',
1152 (parCat[
'compRetrievedTauNight'].size, ))])
1154 inParams[
'PARALPHA'][:] = parCat[
'parAlpha'][0, :]
1155 inParams[
'PARO3'][:] = parCat[
'parO3'][0, :]
1156 inParams[
'PARLNTAUINTERCEPT'][:] = parCat[
'parLnTauIntercept'][0, :]
1157 inParams[
'PARLNTAUSLOPE'][:] = parCat[
'parLnTauSlope'][0, :]
1158 inParams[
'PARLNPWVINTERCEPT'][:] = parCat[
'parLnPwvIntercept'][0, :]
1159 inParams[
'PARLNPWVSLOPE'][:] = parCat[
'parLnPwvSlope'][0, :]
1160 inParams[
'PARLNPWVQUADRATIC'][:] = parCat[
'parLnPwvQuadratic'][0, :]
1161 inParams[
'PARQESYSINTERCEPT'][:] = parCat[
'parQeSysIntercept'][0, :]
1162 inParams[
'COMPQESYSSLOPE'][:] = parCat[
'compQeSysSlope'][0, :]
1163 inParams[
'PARFILTEROFFSET'][:] = parCat[
'parFilterOffset'][0, :]
1164 inParams[
'PARFILTEROFFSETFITFLAG'][:] = parCat[
'parFilterOffsetFitFlag'][0, :]
1165 inParams[
'PARRETRIEVEDLNPWVSCALE'] = parCat[
'parRetrievedLnPwvScale']
1166 inParams[
'PARRETRIEVEDLNPWVOFFSET'] = parCat[
'parRetrievedLnPwvOffset']
1167 inParams[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat[
'parRetrievedLnPwvNightlyOffset'][0, :]
1168 inParams[
'COMPABSTHROUGHPUT'][:] = parCat[
'compAbsThroughput'][0, :]
1169 inParams[
'COMPREFOFFSET'][:] = parCat[
'compRefOffset'][0, :]
1170 inParams[
'COMPREFSIGMA'][:] = parCat[
'compRefSigma'][0, :]
1171 inParams[
'COMPMIRRORCHROMATICITY'][:] = parCat[
'compMirrorChromaticity'][0, :]
1172 inParams[
'MIRRORCHROMATICITYPIVOT'][:] = parCat[
'mirrorChromaticityPivot'][0, :]
1173 inParams[
'COMPAPERCORRPIVOT'][:] = parCat[
'compAperCorrPivot'][0, :]
1174 inParams[
'COMPAPERCORRSLOPE'][:] = parCat[
'compAperCorrSlope'][0, :]
1175 inParams[
'COMPAPERCORRSLOPEERR'][:] = parCat[
'compAperCorrSlopeErr'][0, :]
1176 inParams[
'COMPAPERCORRRANGE'][:] = parCat[
'compAperCorrRange'][0, :]
1177 inParams[
'COMPMODELERREXPTIMEPIVOT'][:] = parCat[
'compModelErrExptimePivot'][0, :]
1178 inParams[
'COMPMODELERRFWHMPIVOT'][:] = parCat[
'compModelErrFwhmPivot'][0, :]
1179 inParams[
'COMPMODELERRSKYPIVOT'][:] = parCat[
'compModelErrSkyPivot'][0, :]
1180 inParams[
'COMPMODELERRPARS'][:] = parCat[
'compModelErrPars'][0, :]
1181 inParams[
'COMPEXPGRAY'][:] = parCat[
'compExpGray'][0, :]
1182 inParams[
'COMPVARGRAY'][:] = parCat[
'compVarGray'][0, :]
1183 inParams[
'COMPNGOODSTARPEREXP'][:] = parCat[
'compNGoodStarPerExp'][0, :]
1184 inParams[
'COMPSIGFGCM'][:] = parCat[
'compSigFgcm'][0, :]
1185 inParams[
'COMPSIGMACAL'][:] = parCat[
'compSigmaCal'][0, :]
1186 inParams[
'COMPRETRIEVEDLNPWV'][:] = parCat[
'compRetrievedLnPwv'][0, :]
1187 inParams[
'COMPRETRIEVEDLNPWVRAW'][:] = parCat[
'compRetrievedLnPwvRaw'][0, :]
1188 inParams[
'COMPRETRIEVEDLNPWVFLAG'][:] = parCat[
'compRetrievedLnPwvFlag'][0, :]
1189 inParams[
'COMPRETRIEVEDTAUNIGHT'][:] = parCat[
'compRetrievedTauNight'][0, :]
1191 inSuperStar = np.zeros(parCat[
'superstarSize'][0, :], dtype=
'f8')
1192 inSuperStar[:, :, :, :] = parCat[
'superstar'][0, :].reshape(inSuperStar.shape)
1194 return (inParInfo, inParams, inSuperStar)
1196 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1198 Persist FGCM datasets through the butler. 1202 butler: `lsst.daf.persistence.Butler` 1203 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle` 1204 Fgcm Fit cycle object 1208 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1210 parSchema = afwTable.Schema()
1213 lutFilterNameString = comma.join([n.decode(
'utf-8')
1214 for n
in parInfo[
'LUTFILTERNAMES'][0]])
1215 fitBandString = comma.join([n.decode(
'utf-8')
1216 for n
in parInfo[
'FITBANDS'][0]])
1217 notFitBandString = comma.join([n.decode(
'utf-8')
1218 for n
in parInfo[
'NOTFITBANDS'][0]])
1220 parSchema = self.
_makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1221 lutFilterNameString, fitBandString, notFitBandString)
1223 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1224 lutFilterNameString, fitBandString, notFitBandString)
1226 butler.put(parCat,
'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1232 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1235 butler.put(flagStarCat,
'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1239 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
1240 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
1243 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1245 butler.put(zptCat,
'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1250 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1252 butler.put(atmCat,
'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1256 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1258 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
1260 butler.put(stdCat,
'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1262 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1263 lutFilterNameString, fitBandString, notFitBandString):
1265 Make the parameter persistence schema 1269 parInfo: `numpy.ndarray` 1270 Parameter information returned by fgcm 1271 pars: `numpy.ndarray` 1272 Parameter values returned by fgcm 1273 parSuperStarFlat: `numpy.array` 1274 Superstar flat values returned by fgcm 1275 lutFilterNameString: `str` 1276 Combined string of all the lutFilterNames 1277 fitBandString: `str` 1278 Combined string of all the fitBands 1279 notFitBandString: `str` 1280 Combined string of all the bands not used in the fit 1284 parSchema: `afwTable.schema` 1287 parSchema = afwTable.Schema()
1290 parSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
1291 parSchema.addField(
'lutFilterNames', type=str, doc=
'LUT Filter names in parameter file',
1292 size=len(lutFilterNameString))
1293 parSchema.addField(
'fitBands', type=str, doc=
'Bands that were fit',
1294 size=len(fitBandString))
1295 parSchema.addField(
'notFitBands', type=str, doc=
'Bands that were not fit',
1296 size=len(notFitBandString))
1297 parSchema.addField(
'lnTauUnit', type=np.float64, doc=
'Step units for ln(AOD)')
1298 parSchema.addField(
'lnTauSlopeUnit', type=np.float64,
1299 doc=
'Step units for ln(AOD) slope')
1300 parSchema.addField(
'alphaUnit', type=np.float64, doc=
'Step units for alpha')
1301 parSchema.addField(
'lnPwvUnit', type=np.float64, doc=
'Step units for ln(pwv)')
1302 parSchema.addField(
'lnPwvSlopeUnit', type=np.float64,
1303 doc=
'Step units for ln(pwv) slope')
1304 parSchema.addField(
'lnPwvQuadraticUnit', type=np.float64,
1305 doc=
'Step units for ln(pwv) quadratic term')
1306 parSchema.addField(
'lnPwvGlobalUnit', type=np.float64,
1307 doc=
'Step units for global ln(pwv) parameters')
1308 parSchema.addField(
'o3Unit', type=np.float64, doc=
'Step units for O3')
1309 parSchema.addField(
'qeSysUnit', type=np.float64, doc=
'Step units for mirror gray')
1310 parSchema.addField(
'filterOffsetUnit', type=np.float64, doc=
'Step units for filter offset')
1311 parSchema.addField(
'hasExternalPwv', type=np.int32, doc=
'Parameters fit using external pwv')
1312 parSchema.addField(
'hasExternalTau', type=np.int32, doc=
'Parameters fit using external tau')
1315 parSchema.addField(
'parAlpha', type=
'ArrayD', doc=
'Alpha parameter vector',
1316 size=pars[
'PARALPHA'].size)
1317 parSchema.addField(
'parO3', type=
'ArrayD', doc=
'O3 parameter vector',
1318 size=pars[
'PARO3'].size)
1319 parSchema.addField(
'parLnTauIntercept', type=
'ArrayD',
1320 doc=
'ln(Tau) intercept parameter vector',
1321 size=pars[
'PARLNTAUINTERCEPT'].size)
1322 parSchema.addField(
'parLnTauSlope', type=
'ArrayD',
1323 doc=
'ln(Tau) slope parameter vector',
1324 size=pars[
'PARLNTAUSLOPE'].size)
1325 parSchema.addField(
'parLnPwvIntercept', type=
'ArrayD', doc=
'ln(pwv) intercept parameter vector',
1326 size=pars[
'PARLNPWVINTERCEPT'].size)
1327 parSchema.addField(
'parLnPwvSlope', type=
'ArrayD', doc=
'ln(pwv) slope parameter vector',
1328 size=pars[
'PARLNPWVSLOPE'].size)
1329 parSchema.addField(
'parLnPwvQuadratic', type=
'ArrayD', doc=
'ln(pwv) quadratic parameter vector',
1330 size=pars[
'PARLNPWVQUADRATIC'].size)
1331 parSchema.addField(
'parQeSysIntercept', type=
'ArrayD', doc=
'Mirror gray intercept parameter vector',
1332 size=pars[
'PARQESYSINTERCEPT'].size)
1333 parSchema.addField(
'compQeSysSlope', type=
'ArrayD', doc=
'Mirror gray slope parameter vector',
1334 size=pars[0][
'COMPQESYSSLOPE'].size)
1335 parSchema.addField(
'parFilterOffset', type=
'ArrayD', doc=
'Filter offset parameter vector',
1336 size=pars[
'PARFILTEROFFSET'].size)
1337 parSchema.addField(
'parFilterOffsetFitFlag', type=
'ArrayI', doc=
'Filter offset parameter fit flag',
1338 size=pars[
'PARFILTEROFFSETFITFLAG'].size)
1339 parSchema.addField(
'parRetrievedLnPwvScale', type=np.float64,
1340 doc=
'Global scale for retrieved ln(pwv)')
1341 parSchema.addField(
'parRetrievedLnPwvOffset', type=np.float64,
1342 doc=
'Global offset for retrieved ln(pwv)')
1343 parSchema.addField(
'parRetrievedLnPwvNightlyOffset', type=
'ArrayD',
1344 doc=
'Nightly offset for retrieved ln(pwv)',
1345 size=pars[
'PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1346 parSchema.addField(
'compAbsThroughput', type=
'ArrayD',
1347 doc=
'Absolute throughput (relative to transmission curves)',
1348 size=pars[
'COMPABSTHROUGHPUT'].size)
1349 parSchema.addField(
'compRefOffset', type=
'ArrayD',
1350 doc=
'Offset between reference stars and calibrated stars',
1351 size=pars[
'COMPREFOFFSET'].size)
1352 parSchema.addField(
'compRefSigma', type=
'ArrayD',
1353 doc=
'Width of reference star/calibrated star distribution',
1354 size=pars[
'COMPREFSIGMA'].size)
1355 parSchema.addField(
'compMirrorChromaticity', type=
'ArrayD',
1356 doc=
'Computed mirror chromaticity terms',
1357 size=pars[
'COMPMIRRORCHROMATICITY'].size)
1358 parSchema.addField(
'mirrorChromaticityPivot', type=
'ArrayD',
1359 doc=
'Mirror chromaticity pivot mjd',
1360 size=pars[
'MIRRORCHROMATICITYPIVOT'].size)
1361 parSchema.addField(
'compAperCorrPivot', type=
'ArrayD', doc=
'Aperture correction pivot',
1362 size=pars[
'COMPAPERCORRPIVOT'].size)
1363 parSchema.addField(
'compAperCorrSlope', type=
'ArrayD', doc=
'Aperture correction slope',
1364 size=pars[
'COMPAPERCORRSLOPE'].size)
1365 parSchema.addField(
'compAperCorrSlopeErr', type=
'ArrayD', doc=
'Aperture correction slope error',
1366 size=pars[
'COMPAPERCORRSLOPEERR'].size)
1367 parSchema.addField(
'compAperCorrRange', type=
'ArrayD', doc=
'Aperture correction range',
1368 size=pars[
'COMPAPERCORRRANGE'].size)
1369 parSchema.addField(
'compModelErrExptimePivot', type=
'ArrayD', doc=
'Model error exptime pivot',
1370 size=pars[
'COMPMODELERREXPTIMEPIVOT'].size)
1371 parSchema.addField(
'compModelErrFwhmPivot', type=
'ArrayD', doc=
'Model error fwhm pivot',
1372 size=pars[
'COMPMODELERRFWHMPIVOT'].size)
1373 parSchema.addField(
'compModelErrSkyPivot', type=
'ArrayD', doc=
'Model error sky pivot',
1374 size=pars[
'COMPMODELERRSKYPIVOT'].size)
1375 parSchema.addField(
'compModelErrPars', type=
'ArrayD', doc=
'Model error parameters',
1376 size=pars[
'COMPMODELERRPARS'].size)
1377 parSchema.addField(
'compExpGray', type=
'ArrayD', doc=
'Computed exposure gray',
1378 size=pars[
'COMPEXPGRAY'].size)
1379 parSchema.addField(
'compVarGray', type=
'ArrayD', doc=
'Computed exposure variance',
1380 size=pars[
'COMPVARGRAY'].size)
1381 parSchema.addField(
'compNGoodStarPerExp', type=
'ArrayI',
1382 doc=
'Computed number of good stars per exposure',
1383 size=pars[
'COMPNGOODSTARPEREXP'].size)
1384 parSchema.addField(
'compSigFgcm', type=
'ArrayD', doc=
'Computed sigma_fgcm (intrinsic repeatability)',
1385 size=pars[
'COMPSIGFGCM'].size)
1386 parSchema.addField(
'compSigmaCal', type=
'ArrayD', doc=
'Computed sigma_cal (systematic error floor)',
1387 size=pars[
'COMPSIGMACAL'].size)
1388 parSchema.addField(
'compRetrievedLnPwv', type=
'ArrayD', doc=
'Retrieved ln(pwv) (smoothed)',
1389 size=pars[
'COMPRETRIEVEDLNPWV'].size)
1390 parSchema.addField(
'compRetrievedLnPwvRaw', type=
'ArrayD', doc=
'Retrieved ln(pwv) (raw)',
1391 size=pars[
'COMPRETRIEVEDLNPWVRAW'].size)
1392 parSchema.addField(
'compRetrievedLnPwvFlag', type=
'ArrayI', doc=
'Retrieved ln(pwv) Flag',
1393 size=pars[
'COMPRETRIEVEDLNPWVFLAG'].size)
1394 parSchema.addField(
'compRetrievedTauNight', type=
'ArrayD', doc=
'Retrieved tau (per night)',
1395 size=pars[
'COMPRETRIEVEDTAUNIGHT'].size)
1397 parSchema.addField(
'superstarSize', type=
'ArrayI', doc=
'Superstar matrix size',
1399 parSchema.addField(
'superstar', type=
'ArrayD', doc=
'Superstar matrix (flattened)',
1400 size=parSuperStarFlat.size)
1404 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1405 lutFilterNameString, fitBandString, notFitBandString):
1407 Make the FGCM parameter catalog for persistence 1411 parSchema: `lsst.afw.table.Schema` 1412 Parameter catalog schema 1413 pars: `numpy.ndarray` 1414 FGCM parameters to put into parCat 1415 parSuperStarFlat: `numpy.array` 1416 FGCM superstar flat array to put into parCat 1417 lutFilterNameString: `str` 1418 Combined string of all the lutFilterNames 1419 fitBandString: `str` 1420 Combined string of all the fitBands 1421 notFitBandString: `str` 1422 Combined string of all the bands not used in the fit 1426 parCat: `afwTable.BasicCatalog` 1427 Atmosphere and instrumental model parameter catalog for persistence 1430 parCat = afwTable.BaseCatalog(parSchema)
1435 rec = parCat.addNew()
1438 rec[
'nCcd'] = parInfo[
'NCCD']
1439 rec[
'lutFilterNames'] = lutFilterNameString
1440 rec[
'fitBands'] = fitBandString
1441 rec[
'notFitBands'] = notFitBandString
1443 rec[
'hasExternalPwv'] = 0
1444 rec[
'hasExternalTau'] = 0
1448 scalarNames = [
'parRetrievedLnPwvScale',
'parRetrievedLnPwvOffset']
1450 arrNames = [
'parAlpha',
'parO3',
'parLnTauIntercept',
'parLnTauSlope',
1451 'parLnPwvIntercept',
'parLnPwvSlope',
'parLnPwvQuadratic',
1452 'parQeSysIntercept',
'compQeSysSlope',
1453 'parRetrievedLnPwvNightlyOffset',
'compAperCorrPivot',
1454 'parFilterOffset',
'parFilterOffsetFitFlag',
1455 'compAbsThroughput',
'compRefOffset',
'compRefSigma',
1456 'compMirrorChromaticity',
'mirrorChromaticityPivot',
1457 'compAperCorrSlope',
'compAperCorrSlopeErr',
'compAperCorrRange',
1458 'compModelErrExptimePivot',
'compModelErrFwhmPivot',
1459 'compModelErrSkyPivot',
'compModelErrPars',
1460 'compExpGray',
'compVarGray',
'compNGoodStarPerExp',
'compSigFgcm',
1462 'compRetrievedLnPwv',
'compRetrievedLnPwvRaw',
'compRetrievedLnPwvFlag',
1463 'compRetrievedTauNight']
1465 for scalarName
in scalarNames:
1466 rec[scalarName] = pars[scalarName.upper()]
1468 for arrName
in arrNames:
1469 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1472 rec[
'superstarSize'][:] = parSuperStarFlat.shape
1473 rec[
'superstar'][:] = parSuperStarFlat.flatten()
1477 def _makeFlagStarSchema(self):
1479 Make the flagged-stars schema 1483 flagStarSchema: `lsst.afw.table.Schema` 1486 flagStarSchema = afwTable.Schema()
1488 flagStarSchema.addField(
'objId', type=np.int32, doc=
'FGCM object id')
1489 flagStarSchema.addField(
'objFlag', type=np.int32, doc=
'FGCM object flag')
1491 return flagStarSchema
1493 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1495 Make the flagged star catalog for persistence 1499 flagStarSchema: `lsst.afw.table.Schema` 1501 flagStarStruct: `numpy.ndarray` 1502 Flagged star structure from fgcm 1506 flagStarCat: `lsst.afw.table.BaseCatalog` 1507 Flagged star catalog for persistence 1510 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1511 flagStarCat.reserve(flagStarStruct.size)
1512 for i
in range(flagStarStruct.size):
1513 flagStarCat.addNew()
1515 flagStarCat[
'objId'][:] = flagStarStruct[
'OBJID']
1516 flagStarCat[
'objFlag'][:] = flagStarStruct[
'OBJFLAG']
def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat, lutFilterNameString, fitBandString, notFitBandString)
def runDataRef(self, butler)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
def _makeFlagStarCat(self, flagStarSchema, flagStarStruct)
def _checkDatasetsExist(self, butler)
def translateFgcmLut(lutCat, filterMap)
def _loadParameters(self, butler)
def computeCcdOffsets(camera, defaultOrientation)
def _persistFgcmDatasets(self, butler, fgcmFitCycle)
def makeStdCat(stdSchema, stdStruct, goodBands)
def _fgcmFitCycle(self, butler)
def _makeFlagStarSchema(self)
def extractReferenceMags(refStars, bands, filterMap)
def __call__(self, butler)
def makeStdSchema(nBands)
def translateVisitCatalog(visitCat)
def writeConfig(self, butler, clobber=False, doBackup=True)
def makeAtmCat(atmSchema, atmStruct)
def _makeParSchema(self, parInfo, pars, parSuperStarFlat, lutFilterNameString, fitBandString, notFitBandString)
def __init__(self, butler=None, kwargs)
def getTargetList(parsedCmd)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def makeZptCat(zptSchema, zpStruct)