Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44import lsst.afw.table as afwTable
46from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
47from .utilities import extractReferenceMags
48from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
49from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
50from .sedterms import SedboundarytermDict, SedtermDict
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57class FgcmFitCycleConfig(pexConfig.Config):
58 """Config for FgcmFitCycle"""
60 bands = pexConfig.ListField(
61 doc="Bands to run calibration (in wavelength order)",
62 dtype=str,
63 default=("NO_DATA",),
64 )
65 fitFlag = pexConfig.ListField(
66 doc=("Flag for which bands are directly constrained in the FGCM fit. "
67 "Bands set to 0 will have the atmosphere constrained from observations "
68 "in other bands on the same night. Must be same length as config.bands, "
69 "and matched band-by-band."),
70 dtype=int,
71 default=(0,),
72 )
73 requiredFlag = pexConfig.ListField(
74 doc=("Flag for which bands are required for a star to be considered a calibration "
75 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
76 "be same length as config.bands, and matched band-by-band."),
77 dtype=int,
78 default=(0,),
79 )
80 filterMap = pexConfig.DictField(
81 doc="Mapping from 'filterName' to band.",
82 keytype=str,
83 itemtype=str,
84 default={},
85 )
86 doReferenceCalibration = pexConfig.Field(
87 doc="Use reference catalog as additional constraint on calibration",
88 dtype=bool,
89 default=True,
90 )
91 refStarSnMin = pexConfig.Field(
92 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
93 dtype=float,
94 default=50.0,
95 )
96 refStarOutlierNSig = pexConfig.Field(
97 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
98 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
99 dtype=float,
100 default=4.0,
101 )
102 applyRefStarColorCuts = pexConfig.Field(
103 doc="Apply color cuts to reference stars?",
104 dtype=bool,
105 default=True,
106 )
107 nCore = pexConfig.Field(
108 doc="Number of cores to use",
109 dtype=int,
110 default=4,
111 )
112 nStarPerRun = pexConfig.Field(
113 doc="Number of stars to run in each chunk",
114 dtype=int,
115 default=200000,
116 )
117 nExpPerRun = pexConfig.Field(
118 doc="Number of exposures to run in each chunk",
119 dtype=int,
120 default=1000,
121 )
122 reserveFraction = pexConfig.Field(
123 doc="Fraction of stars to reserve for testing",
124 dtype=float,
125 default=0.1,
126 )
127 freezeStdAtmosphere = pexConfig.Field(
128 doc="Freeze atmosphere parameters to standard (for testing)",
129 dtype=bool,
130 default=False,
131 )
132 precomputeSuperStarInitialCycle = pexConfig.Field(
133 doc="Precompute superstar flat for initial cycle",
134 dtype=bool,
135 default=False,
136 )
137 superStarSubCcd = pexConfig.Field(
138 doc="Compute superstar flat on sub-ccd scale",
139 dtype=bool,
140 default=True,
141 )
142 superStarSubCcdChebyshevOrder = pexConfig.Field(
143 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
144 "Global default is first-order polynomials, and should be overridden "
145 "on a camera-by-camera basis depending on the ISR."),
146 dtype=int,
147 default=1,
148 )
149 superStarSubCcdTriangular = pexConfig.Field(
150 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
151 "suppress high-order cross terms?"),
152 dtype=bool,
153 default=False,
154 )
155 superStarSigmaClip = pexConfig.Field(
156 doc="Number of sigma to clip outliers when selecting for superstar flats",
157 dtype=float,
158 default=5.0,
159 )
160 ccdGraySubCcd = pexConfig.Field(
161 doc="Compute CCD gray terms on sub-ccd scale",
162 dtype=bool,
163 default=False,
164 )
165 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
166 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
167 dtype=int,
168 default=1,
169 )
170 ccdGraySubCcdTriangular = pexConfig.Field(
171 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
172 "suppress high-order cross terms?"),
173 dtype=bool,
174 default=True,
175 )
176 cycleNumber = pexConfig.Field(
177 doc=("FGCM fit cycle number. This is automatically incremented after each run "
178 "and stage of outlier rejection. See cookbook for details."),
179 dtype=int,
180 default=None,
181 )
182 isFinalCycle = pexConfig.Field(
183 doc=("Is this the final cycle of the fitting? Will automatically compute final "
184 "selection of stars and photometric exposures, and will output zeropoints "
185 "and standard stars for use in fgcmOutputProducts"),
186 dtype=bool,
187 default=False,
188 )
189 maxIterBeforeFinalCycle = pexConfig.Field(
190 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
191 "will always be 0 in the final cycle for cleanup and final selection."),
192 dtype=int,
193 default=50,
194 )
195 utBoundary = pexConfig.Field(
196 doc="Boundary (in UTC) from day-to-day",
197 dtype=float,
198 default=None,
199 )
200 washMjds = pexConfig.ListField(
201 doc="Mirror wash MJDs",
202 dtype=float,
203 default=(0.0,),
204 )
205 epochMjds = pexConfig.ListField(
206 doc="Epoch boundaries in MJD",
207 dtype=float,
208 default=(0.0,),
209 )
210 minObsPerBand = pexConfig.Field(
211 doc="Minimum good observations per band",
212 dtype=int,
213 default=2,
214 )
215 # TODO: When DM-16511 is done, it will be possible to get the
216 # telescope latitude directly from the camera.
217 latitude = pexConfig.Field(
218 doc="Observatory latitude",
219 dtype=float,
220 default=None,
221 )
222 pixelScale = pexConfig.Field(
223 doc="Pixel scale (arcsec/pixel) (temporary)",
224 dtype=float,
225 deprecated=("This field is no longer used, and has been deprecated by DM-16490. "
226 "It will be removed after v19."),
227 optional=True,
228 )
229 brightObsGrayMax = pexConfig.Field(
230 doc="Maximum gray extinction to be considered bright observation",
231 dtype=float,
232 default=0.15,
233 )
234 minStarPerCcd = pexConfig.Field(
235 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
236 "CCDs with fewer stars will have their calibration estimated from other "
237 "CCDs in the same visit, with zeropoint error increased accordingly."),
238 dtype=int,
239 default=5,
240 )
241 minCcdPerExp = pexConfig.Field(
242 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
243 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
244 dtype=int,
245 default=5,
246 )
247 maxCcdGrayErr = pexConfig.Field(
248 doc="Maximum error on CCD gray offset to be considered photometric",
249 dtype=float,
250 default=0.05,
251 )
252 minStarPerExp = pexConfig.Field(
253 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
254 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
255 dtype=int,
256 default=600,
257 )
258 minExpPerNight = pexConfig.Field(
259 doc="Minimum number of good exposures/visits to consider a partly photometric night",
260 dtype=int,
261 default=10,
262 )
263 expGrayInitialCut = pexConfig.Field(
264 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
265 "observations."),
266 dtype=float,
267 default=-0.25,
268 )
269 expGrayPhotometricCut = pexConfig.ListField(
270 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
271 "Must be same length as config.bands, and matched band-by-band."),
272 dtype=float,
273 default=(0.0,),
274 )
275 expGrayHighCut = pexConfig.ListField(
276 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
277 "Must be same length as config.bands, and matched band-by-band."),
278 dtype=float,
279 default=(0.0,),
280 )
281 expGrayRecoverCut = pexConfig.Field(
282 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
283 "Visits with more gray extinction will only get CCD zeropoints if there are "
284 "sufficient star observations (minStarPerCcd) on that CCD."),
285 dtype=float,
286 default=-1.0,
287 )
288 expVarGrayPhotometricCut = pexConfig.Field(
289 doc="Maximum exposure variance to be considered possibly photometric",
290 dtype=float,
291 default=0.0005,
292 )
293 expGrayErrRecoverCut = pexConfig.Field(
294 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
295 "Visits with more gray variance will only get CCD zeropoints if there are "
296 "sufficient star observations (minStarPerCcd) on that CCD."),
297 dtype=float,
298 default=0.05,
299 )
300 aperCorrFitNBins = pexConfig.Field(
301 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
302 "no fit will be performed, and the config.aperCorrInputSlopes will be "
303 "used if available."),
304 dtype=int,
305 default=10,
306 )
307 aperCorrInputSlopes = pexConfig.ListField(
308 doc=("Aperture correction input slope parameters. These are used on the first "
309 "fit iteration, and aperture correction parameters will be updated from "
310 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
311 "when there is insufficient data to fit the parameters (e.g. tract mode). "
312 "If set, must be same length as config.bands, and matched band-by-band."),
313 dtype=float,
314 default=[],
315 )
316 sedFudgeFactors = pexConfig.ListField(
317 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
318 "config.bands, and matched band-by-band."),
319 dtype=float,
320 default=(0,),
321 deprecated=("This field has been deprecated and will be removed after v20. "
322 "Please use sedSlopeTermMap and sedSlopeMap."),
323 )
324 sedboundaryterms = pexConfig.ConfigField(
325 doc="Mapping from bands to SED boundary term names used is sedterms.",
326 dtype=SedboundarytermDict,
327 )
328 sedterms = pexConfig.ConfigField(
329 doc="Mapping from terms to bands for fgcm linear SED approximations.",
330 dtype=SedtermDict,
331 )
332 sigFgcmMaxErr = pexConfig.Field(
333 doc="Maximum mag error for fitting sigma_FGCM",
334 dtype=float,
335 default=0.01,
336 )
337 sigFgcmMaxEGray = pexConfig.ListField(
338 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
339 "May be 1 element (same for all bands) or the same length as config.bands."),
340 dtype=float,
341 default=(0.05,),
342 )
343 ccdGrayMaxStarErr = pexConfig.Field(
344 doc="Maximum error on a star observation to use in ccd gray computation",
345 dtype=float,
346 default=0.10,
347 )
348 approxThroughput = pexConfig.ListField(
349 doc=("Approximate overall throughput at start of calibration observations. "
350 "May be 1 element (same for all bands) or the same length as config.bands."),
351 dtype=float,
352 default=(1.0, ),
353 )
354 sigmaCalRange = pexConfig.ListField(
355 doc="Allowed range for systematic error floor estimation",
356 dtype=float,
357 default=(0.001, 0.003),
358 )
359 sigmaCalFitPercentile = pexConfig.ListField(
360 doc="Magnitude percentile range to fit systematic error floor",
361 dtype=float,
362 default=(0.05, 0.15),
363 )
364 sigmaCalPlotPercentile = pexConfig.ListField(
365 doc="Magnitude percentile range to plot systematic error floor",
366 dtype=float,
367 default=(0.05, 0.95),
368 )
369 sigma0Phot = pexConfig.Field(
370 doc="Systematic error floor for all zeropoints",
371 dtype=float,
372 default=0.003,
373 )
374 mapLongitudeRef = pexConfig.Field(
375 doc="Reference longitude for plotting maps",
376 dtype=float,
377 default=0.0,
378 )
379 mapNSide = pexConfig.Field(
380 doc="Healpix nside for plotting maps",
381 dtype=int,
382 default=256,
383 )
384 outfileBase = pexConfig.Field(
385 doc="Filename start for plot output files",
386 dtype=str,
387 default=None,
388 )
389 starColorCuts = pexConfig.ListField(
390 doc="Encoded star-color cuts (to be cleaned up)",
391 dtype=str,
392 default=("NO_DATA",),
393 )
394 colorSplitIndices = pexConfig.ListField(
395 doc="Band indices to use to split stars by color",
396 dtype=int,
397 default=None,
398 )
399 modelMagErrors = pexConfig.Field(
400 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
401 dtype=bool,
402 default=True,
403 )
404 useQuadraticPwv = pexConfig.Field(
405 doc="Model PWV with a quadratic term for variation through the night?",
406 dtype=bool,
407 default=False,
408 )
409 instrumentParsPerBand = pexConfig.Field(
410 doc=("Model instrumental parameters per band? "
411 "Otherwise, instrumental parameters (QE changes with time) are "
412 "shared among all bands."),
413 dtype=bool,
414 default=False,
415 )
416 instrumentSlopeMinDeltaT = pexConfig.Field(
417 doc=("Minimum time change (in days) between observations to use in constraining "
418 "instrument slope."),
419 dtype=float,
420 default=20.0,
421 )
422 fitMirrorChromaticity = pexConfig.Field(
423 doc="Fit (intraband) mirror chromatic term?",
424 dtype=bool,
425 default=False,
426 )
427 coatingMjds = pexConfig.ListField(
428 doc="Mirror coating dates in MJD",
429 dtype=float,
430 default=(0.0,),
431 )
432 outputStandardsBeforeFinalCycle = pexConfig.Field(
433 doc="Output standard stars prior to final cycle? Used in debugging.",
434 dtype=bool,
435 default=False,
436 )
437 outputZeropointsBeforeFinalCycle = pexConfig.Field(
438 doc="Output standard stars prior to final cycle? Used in debugging.",
439 dtype=bool,
440 default=False,
441 )
442 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
443 doc=("Use star repeatability (instead of exposures) for computing photometric "
444 "cuts? Recommended for tract mode or bands with few exposures. "
445 "May be 1 element (same for all bands) or the same length as config.bands."),
446 dtype=bool,
447 default=(False,),
448 )
449 autoPhotometricCutNSig = pexConfig.Field(
450 doc=("Number of sigma for automatic computation of (low) photometric cut. "
451 "Cut is based on exposure gray width (per band), unless "
452 "useRepeatabilityForExpGrayCuts is set, in which case the star "
453 "repeatability is used (also per band)."),
454 dtype=float,
455 default=3.0,
456 )
457 autoHighCutNSig = pexConfig.Field(
458 doc=("Number of sigma for automatic computation of (high) outlier cut. "
459 "Cut is based on exposure gray width (per band), unless "
460 "useRepeatabilityForExpGrayCuts is set, in which case the star "
461 "repeatability is used (also per band)."),
462 dtype=float,
463 default=4.0,
464 )
465 quietMode = pexConfig.Field(
466 doc="Be less verbose with logging.",
467 dtype=bool,
468 default=False,
469 )
471 def setDefaults(self):
472 pass
475class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
476 """Subclass of TaskRunner for fgcmFitCycleTask
478 fgcmFitCycleTask.run() takes one argument, the butler, and uses
479 stars and visits previously extracted from dataRefs by
480 fgcmBuildStars.
481 This Runner does not perform any dataRef parallelization, but the FGCM
482 code called by the Task uses python multiprocessing (see the "ncores"
483 config option).
484 """
486 @staticmethod
487 def getTargetList(parsedCmd):
488 """
489 Return a list with one element, the butler.
490 """
491 return [parsedCmd.butler]
493 def __call__(self, butler):
494 """
495 Parameters
496 ----------
497 butler: `lsst.daf.persistence.Butler`
499 Returns
500 -------
501 exitStatus: `list` with `pipeBase.Struct`
502 exitStatus (0: success; 1: failure)
503 """
505 task = self.TaskClass(config=self.config, log=self.log)
507 exitStatus = 0
508 if self.doRaise:
509 task.runDataRef(butler)
510 else:
511 try:
512 task.runDataRef(butler)
513 except Exception as e:
514 exitStatus = 1
515 task.log.fatal("Failed: %s" % e)
516 if not isinstance(e, pipeBase.TaskError):
517 traceback.print_exc(file=sys.stderr)
519 task.writeMetadata(butler)
521 # The task does not return any results:
522 return [pipeBase.Struct(exitStatus=exitStatus)]
524 def run(self, parsedCmd):
525 """
526 Run the task, with no multiprocessing
528 Parameters
529 ----------
530 parsedCmd: ArgumentParser parsed command line
531 """
533 resultList = []
535 if self.precall(parsedCmd):
536 targetList = self.getTargetList(parsedCmd)
537 # make sure that we only get 1
538 resultList = self(targetList[0])
540 return resultList
543class FgcmFitCycleTask(pipeBase.CmdLineTask):
544 """
545 Run Single fit cycle for FGCM global calibration
546 """
548 ConfigClass = FgcmFitCycleConfig
549 RunnerClass = FgcmFitCycleRunner
550 _DefaultName = "fgcmFitCycle"
552 def __init__(self, butler=None, **kwargs):
553 """
554 Instantiate an fgcmFitCycle.
556 Parameters
557 ----------
558 butler : `lsst.daf.persistence.Butler`
559 """
561 pipeBase.CmdLineTask.__init__(self, **kwargs)
563 # no saving of metadata for now
564 def _getMetadataName(self):
565 return None
567 @pipeBase.timeMethod
568 def runDataRef(self, butler):
569 """
570 Run a single fit cycle for FGCM
572 Parameters
573 ----------
574 butler: `lsst.daf.persistence.Butler`
575 """
577 self._fgcmFitCycle(butler)
579 def writeConfig(self, butler, clobber=False, doBackup=True):
580 """Write the configuration used for processing the data, or check that an existing
581 one is equal to the new one if present. This is an override of the regular
582 version from pipe_base that knows about fgcmcycle.
584 Parameters
585 ----------
586 butler : `lsst.daf.persistence.Butler`
587 Data butler used to write the config. The config is written to dataset type
588 `CmdLineTask._getConfigName`.
589 clobber : `bool`, optional
590 A boolean flag that controls what happens if a config already has been saved:
591 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
592 - `False`: raise `TaskError` if this config does not match the existing config.
593 doBackup : `bool`, optional
594 Set to `True` to backup the config files if clobbering.
595 """
596 configName = self._getConfigName()
597 if configName is None:
598 return
599 if clobber:
600 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
601 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
602 # this may be subject to a race condition; see #2789
603 try:
604 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
605 except Exception as exc:
606 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
607 (configName, exc))
609 def logConfigMismatch(msg):
610 self.log.fatal("Comparing configuration: %s", msg)
612 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
613 raise pipeBase.TaskError(
614 ("Config does not match existing task config %r on disk; tasks configurations " +
615 "must be consistent within the same output repo (override with --clobber-config)") %
616 (configName,))
617 else:
618 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
620 def _fgcmFitCycle(self, butler):
621 """
622 Run the fit cycle
624 Parameters
625 ----------
626 butler: `lsst.daf.persistence.Butler`
627 """
629 self._checkDatasetsExist(butler)
631 # Set defaults on whether to output standards and zeropoints
632 self.maxIter = self.config.maxIterBeforeFinalCycle
633 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
634 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
635 self.resetFitParameters = True
637 if self.config.isFinalCycle:
638 # This is the final fit cycle, so we do not want to reset fit
639 # parameters, we want to run a final "clean-up" with 0 fit iterations,
640 # and we always want to output standards and zeropoints
641 self.maxIter = 0
642 self.outputStandards = True
643 self.outputZeropoints = True
644 self.resetFitParameters = False
646 camera = butler.get('camera')
647 configDict = makeConfigDict(self.config, self.log, camera,
648 self.maxIter, self.resetFitParameters,
649 self.outputZeropoints)
651 lutCat = butler.get('fgcmLookUpTable')
652 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat, dict(self.config.filterMap))
653 del lutCat
655 # next we need the exposure/visit information
657 # fgcmExpInfo = self._loadVisitCatalog(butler)
658 visitCat = butler.get('fgcmVisitCatalog')
659 fgcmExpInfo = translateVisitCatalog(visitCat)
660 del visitCat
662 # Use the first orientation.
663 # TODO: DM-21215 will generalize to arbitrary camera orientations
664 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
666 noFitsDict = {'lutIndex': lutIndexVals,
667 'lutStd': lutStd,
668 'expInfo': fgcmExpInfo,
669 'ccdOffsets': ccdOffsets}
671 # set up the fitter object
672 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
673 noFitsDict=noFitsDict, noOutput=True)
675 # create the parameter object
676 if (fgcmFitCycle.initialCycle):
677 # cycle = 0, initial cycle
678 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
679 fgcmLut,
680 fgcmExpInfo)
681 else:
682 inParInfo, inParams, inSuperStar = self._loadParameters(butler)
683 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
684 fgcmExpInfo,
685 inParInfo,
686 inParams,
687 inSuperStar)
689 lastCycle = configDict['cycleNumber'] - 1
691 # set up the stars...
692 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
694 starObs = butler.get('fgcmStarObservations')
695 starIds = butler.get('fgcmStarIds')
696 starIndices = butler.get('fgcmStarIndices')
698 # grab the flagged stars if available
699 if butler.datasetExists('fgcmFlaggedStars', fgcmcycle=lastCycle):
700 flaggedStars = butler.get('fgcmFlaggedStars', fgcmcycle=lastCycle)
701 flagId = flaggedStars['objId'][:]
702 flagFlag = flaggedStars['objFlag'][:]
703 else:
704 flagId = None
705 flagFlag = None
707 if self.config.doReferenceCalibration:
708 refStars = butler.get('fgcmReferenceStars')
710 refMag, refMagErr = extractReferenceMags(refStars,
711 self.config.bands,
712 self.config.filterMap)
713 refId = refStars['fgcm_id'][:]
714 else:
715 refId = None
716 refMag = None
717 refMagErr = None
719 # match star observations to visits
720 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
721 # actually be transferred into fgcm using the indexing below.
722 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
724 # The fgcmStars.loadStars method will copy all the star information into
725 # special shared memory objects that will not blow up the memory usage when
726 # used with python multiprocessing. Once all the numbers are copied,
727 # it is necessary to release all references to the objects that previously
728 # stored the data to ensure that the garbage collector can clear the memory,
729 # and ensure that this memory is not copied when multiprocessing kicks in.
731 # We determine the conversion from the native units (typically radians) to
732 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
733 # numpy arrays rather than Angles, which would we approximately 600x slower.
734 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
736 fgcmStars.loadStars(fgcmPars,
737 starObs['visit'][starIndices['obsIndex']],
738 starObs['ccd'][starIndices['obsIndex']],
739 starObs['ra'][starIndices['obsIndex']] * conv,
740 starObs['dec'][starIndices['obsIndex']] * conv,
741 starObs['instMag'][starIndices['obsIndex']],
742 starObs['instMagErr'][starIndices['obsIndex']],
743 fgcmExpInfo['FILTERNAME'][visitIndex],
744 starIds['fgcm_id'][:],
745 starIds['ra'][:],
746 starIds['dec'][:],
747 starIds['obsArrIndex'][:],
748 starIds['nObs'][:],
749 obsX=starObs['x'][starIndices['obsIndex']],
750 obsY=starObs['y'][starIndices['obsIndex']],
751 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
752 refID=refId,
753 refMag=refMag,
754 refMagErr=refMagErr,
755 flagID=flagId,
756 flagFlag=flagFlag,
757 computeNobs=True)
759 # Release all references to temporary objects holding star data (see above)
760 starObs = None
761 starIds = None
762 starIndices = None
763 flagId = None
764 flagFlag = None
765 flaggedStars = None
766 refStars = None
768 # and set the bits in the cycle object
769 fgcmFitCycle.setLUT(fgcmLut)
770 fgcmFitCycle.setStars(fgcmStars)
771 fgcmFitCycle.setPars(fgcmPars)
773 # finish the setup
774 fgcmFitCycle.finishSetup()
776 # and run
777 fgcmFitCycle.run()
779 ##################
780 # Persistance
781 ##################
783 self._persistFgcmDatasets(butler, fgcmFitCycle)
785 # Output the config for the next cycle
786 # We need to make a copy since the input one has been frozen
788 outConfig = copy.copy(self.config)
789 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
790 precomputeSuperStarInitialCycle=False,
791 freezeStdAtmosphere=False,
792 expGrayPhotometricCut=fgcmFitCycle.updatedPhotometricCut,
793 expGrayHighCut=fgcmFitCycle.updatedHighCut)
794 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
795 outConfig.cycleNumber)
796 outConfig.save(configFileName)
798 if self.config.isFinalCycle == 1:
799 # We are done, ready to output products
800 self.log.info("Everything is in place to run fgcmOutputProducts.py")
801 else:
802 self.log.info("Saved config for next cycle to %s" % (configFileName))
803 self.log.info("Be sure to look at:")
804 self.log.info(" config.expGrayPhotometricCut")
805 self.log.info(" config.expGrayHighCut")
806 self.log.info("If you are satisfied with the fit, please set:")
807 self.log.info(" config.isFinalCycle = True")
809 def _checkDatasetsExist(self, butler):
810 """
811 Check if necessary datasets exist to run fgcmFitCycle
813 Parameters
814 ----------
815 butler: `lsst.daf.persistence.Butler`
817 Raises
818 ------
819 RuntimeError
820 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
821 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
822 If cycleNumber > 0, then also checks for fgcmFitParameters,
823 fgcmFlaggedStars.
824 """
826 if not butler.datasetExists('fgcmVisitCatalog'):
827 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
828 if not butler.datasetExists('fgcmStarObservations'):
829 raise RuntimeError("Could not find fgcmStarObservations in repo!")
830 if not butler.datasetExists('fgcmStarIds'):
831 raise RuntimeError("Could not find fgcmStarIds in repo!")
832 if not butler.datasetExists('fgcmStarIndices'):
833 raise RuntimeError("Could not find fgcmStarIndices in repo!")
834 if not butler.datasetExists('fgcmLookUpTable'):
835 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
837 # Need additional datasets if we are not the initial cycle
838 if (self.config.cycleNumber > 0):
839 if not butler.datasetExists('fgcmFitParameters',
840 fgcmcycle=self.config.cycleNumber-1):
841 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
842 (self.config.cycleNumber-1))
843 if not butler.datasetExists('fgcmFlaggedStars',
844 fgcmcycle=self.config.cycleNumber-1):
845 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
846 (self.config.cycleNumber-1))
848 # And additional dataset if we want reference calibration
849 if self.config.doReferenceCalibration:
850 if not butler.datasetExists('fgcmReferenceStars'):
851 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
852 "doReferenceCalibration is True.")
854 def _loadParameters(self, butler):
855 """
856 Load FGCM parameters from a previous fit cycle
858 Parameters
859 ----------
860 butler: `lsst.daf.persistence.Butler`
862 Returns
863 -------
864 inParInfo: `numpy.ndarray`
865 Numpy array parameter information formatted for input to fgcm
866 inParameters: `numpy.ndarray`
867 Numpy array parameter values formatted for input to fgcm
868 inSuperStar: `numpy.array`
869 Superstar flat formatted for input to fgcm
870 """
872 # note that we already checked that this is available
873 parCat = butler.get('fgcmFitParameters', fgcmcycle=self.config.cycleNumber-1)
875 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
876 parFitBands = np.array(parCat[0]['fitBands'].split(','))
877 parNotFitBands = np.array(parCat[0]['notFitBands'].split(','))
879 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
880 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
881 parLutFilterNames.size),
882 ('FITBANDS', parFitBands.dtype.str, parFitBands.size),
883 ('NOTFITBANDS', parNotFitBands.dtype.str, parNotFitBands.size),
884 ('LNTAUUNIT', 'f8'),
885 ('LNTAUSLOPEUNIT', 'f8'),
886 ('ALPHAUNIT', 'f8'),
887 ('LNPWVUNIT', 'f8'),
888 ('LNPWVSLOPEUNIT', 'f8'),
889 ('LNPWVQUADRATICUNIT', 'f8'),
890 ('LNPWVGLOBALUNIT', 'f8'),
891 ('O3UNIT', 'f8'),
892 ('QESYSUNIT', 'f8'),
893 ('FILTEROFFSETUNIT', 'f8'),
894 ('HASEXTERNALPWV', 'i2'),
895 ('HASEXTERNALTAU', 'i2')])
896 inParInfo['NCCD'] = parCat['nCcd']
897 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
898 inParInfo['FITBANDS'][:] = parFitBands
899 inParInfo['NOTFITBANDS'][:] = parNotFitBands
900 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
901 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
903 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', parCat['parAlpha'].size),
904 ('PARO3', 'f8', parCat['parO3'].size),
905 ('PARLNTAUINTERCEPT', 'f8',
906 parCat['parLnTauIntercept'].size),
907 ('PARLNTAUSLOPE', 'f8',
908 parCat['parLnTauSlope'].size),
909 ('PARLNPWVINTERCEPT', 'f8',
910 parCat['parLnPwvIntercept'].size),
911 ('PARLNPWVSLOPE', 'f8',
912 parCat['parLnPwvSlope'].size),
913 ('PARLNPWVQUADRATIC', 'f8',
914 parCat['parLnPwvQuadratic'].size),
915 ('PARQESYSINTERCEPT', 'f8',
916 parCat['parQeSysIntercept'].size),
917 ('COMPQESYSSLOPE', 'f8',
918 parCat['compQeSysSlope'].size),
919 ('PARFILTEROFFSET', 'f8',
920 parCat['parFilterOffset'].size),
921 ('PARFILTEROFFSETFITFLAG', 'i2',
922 parCat['parFilterOffsetFitFlag'].size),
923 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
924 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
925 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
926 parCat['parRetrievedLnPwvNightlyOffset'].size),
927 ('COMPABSTHROUGHPUT', 'f8',
928 parCat['compAbsThroughput'].size),
929 ('COMPREFOFFSET', 'f8',
930 parCat['compRefOffset'].size),
931 ('COMPREFSIGMA', 'f8',
932 parCat['compRefSigma'].size),
933 ('COMPMIRRORCHROMATICITY', 'f8',
934 parCat['compMirrorChromaticity'].size),
935 ('MIRRORCHROMATICITYPIVOT', 'f8',
936 parCat['mirrorChromaticityPivot'].size),
937 ('COMPAPERCORRPIVOT', 'f8',
938 parCat['compAperCorrPivot'].size),
939 ('COMPAPERCORRSLOPE', 'f8',
940 parCat['compAperCorrSlope'].size),
941 ('COMPAPERCORRSLOPEERR', 'f8',
942 parCat['compAperCorrSlopeErr'].size),
943 ('COMPAPERCORRRANGE', 'f8',
944 parCat['compAperCorrRange'].size),
945 ('COMPMODELERREXPTIMEPIVOT', 'f8',
946 parCat['compModelErrExptimePivot'].size),
947 ('COMPMODELERRFWHMPIVOT', 'f8',
948 parCat['compModelErrFwhmPivot'].size),
949 ('COMPMODELERRSKYPIVOT', 'f8',
950 parCat['compModelErrSkyPivot'].size),
951 ('COMPMODELERRPARS', 'f8',
952 parCat['compModelErrPars'].size),
953 ('COMPEXPGRAY', 'f8',
954 parCat['compExpGray'].size),
955 ('COMPVARGRAY', 'f8',
956 parCat['compVarGray'].size),
957 ('COMPNGOODSTARPEREXP', 'i4',
958 parCat['compNGoodStarPerExp'].size),
959 ('COMPSIGFGCM', 'f8',
960 parCat['compSigFgcm'].size),
961 ('COMPSIGMACAL', 'f8',
962 parCat['compSigmaCal'].size),
963 ('COMPRETRIEVEDLNPWV', 'f8',
964 parCat['compRetrievedLnPwv'].size),
965 ('COMPRETRIEVEDLNPWVRAW', 'f8',
966 parCat['compRetrievedLnPwvRaw'].size),
967 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
968 parCat['compRetrievedLnPwvFlag'].size),
969 ('COMPRETRIEVEDTAUNIGHT', 'f8',
970 parCat['compRetrievedTauNight'].size)])
972 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
973 inParams['PARO3'][:] = parCat['parO3'][0, :]
974 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
975 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
976 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
977 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
978 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
979 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
980 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
981 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
982 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
983 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
984 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
985 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
986 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
987 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
988 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
989 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
990 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
991 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
992 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
993 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
994 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
995 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
996 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
997 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
998 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
999 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1000 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1001 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1002 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1003 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1004 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1005 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1006 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1007 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1009 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1010 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1012 return (inParInfo, inParams, inSuperStar)
1014 def _persistFgcmDatasets(self, butler, fgcmFitCycle):
1015 """
1016 Persist FGCM datasets through the butler.
1018 Parameters
1019 ----------
1020 butler: `lsst.daf.persistence.Butler`
1021 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1022 Fgcm Fit cycle object
1023 """
1025 # Save the parameters
1026 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1028 parSchema = afwTable.Schema()
1030 comma = ','
1031 lutFilterNameString = comma.join([n.decode('utf-8')
1032 for n in parInfo['LUTFILTERNAMES'][0]])
1033 fitBandString = comma.join([n.decode('utf-8')
1034 for n in parInfo['FITBANDS'][0]])
1035 notFitBandString = comma.join([n.decode('utf-8')
1036 for n in parInfo['NOTFITBANDS'][0]])
1038 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1039 lutFilterNameString, fitBandString, notFitBandString)
1040 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1041 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1042 lutFilterNameString, fitBandString, notFitBandString)
1044 butler.put(parCat, 'fgcmFitParameters', fgcmcycle=self.config.cycleNumber)
1046 # Save the indices of the flagged stars
1047 # (stars that have been (a) reserved from the fit for testing and
1048 # (b) bad stars that have failed quality checks.)
1049 flagStarSchema = self._makeFlagStarSchema()
1050 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1051 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1053 butler.put(flagStarCat, 'fgcmFlaggedStars', fgcmcycle=self.config.cycleNumber)
1055 # Save the zeropoint information and atmospheres only if desired
1056 if self.outputZeropoints:
1057 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1058 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1060 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1061 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1063 butler.put(zptCat, 'fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
1065 # Save atmosphere values
1066 # These are generated by the same code that generates zeropoints
1067 atmSchema = makeAtmSchema()
1068 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1070 butler.put(atmCat, 'fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)
1072 # Save the standard stars (if configured)
1073 if self.outputStandards:
1074 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1075 stdSchema = makeStdSchema(len(goodBands))
1076 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1078 butler.put(stdCat, 'fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
1080 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1081 lutFilterNameString, fitBandString, notFitBandString):
1082 """
1083 Make the parameter persistence schema
1085 Parameters
1086 ----------
1087 parInfo: `numpy.ndarray`
1088 Parameter information returned by fgcm
1089 pars: `numpy.ndarray`
1090 Parameter values returned by fgcm
1091 parSuperStarFlat: `numpy.array`
1092 Superstar flat values returned by fgcm
1093 lutFilterNameString: `str`
1094 Combined string of all the lutFilterNames
1095 fitBandString: `str`
1096 Combined string of all the fitBands
1097 notFitBandString: `str`
1098 Combined string of all the bands not used in the fit
1100 Returns
1101 -------
1102 parSchema: `afwTable.schema`
1103 """
1105 parSchema = afwTable.Schema()
1107 # parameter info section
1108 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1109 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1110 size=len(lutFilterNameString))
1111 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1112 size=len(fitBandString))
1113 parSchema.addField('notFitBands', type=str, doc='Bands that were not fit',
1114 size=len(notFitBandString))
1115 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1116 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1117 doc='Step units for ln(AOD) slope')
1118 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1119 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1120 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1121 doc='Step units for ln(pwv) slope')
1122 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1123 doc='Step units for ln(pwv) quadratic term')
1124 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1125 doc='Step units for global ln(pwv) parameters')
1126 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1127 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1128 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1129 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1130 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1132 # parameter section
1133 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1134 size=pars['PARALPHA'].size)
1135 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1136 size=pars['PARO3'].size)
1137 parSchema.addField('parLnTauIntercept', type='ArrayD',
1138 doc='ln(Tau) intercept parameter vector',
1139 size=pars['PARLNTAUINTERCEPT'].size)
1140 parSchema.addField('parLnTauSlope', type='ArrayD',
1141 doc='ln(Tau) slope parameter vector',
1142 size=pars['PARLNTAUSLOPE'].size)
1143 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1144 size=pars['PARLNPWVINTERCEPT'].size)
1145 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1146 size=pars['PARLNPWVSLOPE'].size)
1147 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1148 size=pars['PARLNPWVQUADRATIC'].size)
1149 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1150 size=pars['PARQESYSINTERCEPT'].size)
1151 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1152 size=pars[0]['COMPQESYSSLOPE'].size)
1153 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1154 size=pars['PARFILTEROFFSET'].size)
1155 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1156 size=pars['PARFILTEROFFSETFITFLAG'].size)
1157 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1158 doc='Global scale for retrieved ln(pwv)')
1159 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1160 doc='Global offset for retrieved ln(pwv)')
1161 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1162 doc='Nightly offset for retrieved ln(pwv)',
1163 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1164 parSchema.addField('compAbsThroughput', type='ArrayD',
1165 doc='Absolute throughput (relative to transmission curves)',
1166 size=pars['COMPABSTHROUGHPUT'].size)
1167 parSchema.addField('compRefOffset', type='ArrayD',
1168 doc='Offset between reference stars and calibrated stars',
1169 size=pars['COMPREFOFFSET'].size)
1170 parSchema.addField('compRefSigma', type='ArrayD',
1171 doc='Width of reference star/calibrated star distribution',
1172 size=pars['COMPREFSIGMA'].size)
1173 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1174 doc='Computed mirror chromaticity terms',
1175 size=pars['COMPMIRRORCHROMATICITY'].size)
1176 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1177 doc='Mirror chromaticity pivot mjd',
1178 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1179 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1180 size=pars['COMPAPERCORRPIVOT'].size)
1181 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1182 size=pars['COMPAPERCORRSLOPE'].size)
1183 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1184 size=pars['COMPAPERCORRSLOPEERR'].size)
1185 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1186 size=pars['COMPAPERCORRRANGE'].size)
1187 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1188 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1189 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1190 size=pars['COMPMODELERRFWHMPIVOT'].size)
1191 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1192 size=pars['COMPMODELERRSKYPIVOT'].size)
1193 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1194 size=pars['COMPMODELERRPARS'].size)
1195 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1196 size=pars['COMPEXPGRAY'].size)
1197 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1198 size=pars['COMPVARGRAY'].size)
1199 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1200 doc='Computed number of good stars per exposure',
1201 size=pars['COMPNGOODSTARPEREXP'].size)
1202 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1203 size=pars['COMPSIGFGCM'].size)
1204 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1205 size=pars['COMPSIGMACAL'].size)
1206 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1207 size=pars['COMPRETRIEVEDLNPWV'].size)
1208 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1209 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1210 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1211 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1212 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1213 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1214 # superstarflat section
1215 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1216 size=4)
1217 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1218 size=parSuperStarFlat.size)
1220 return parSchema
1222 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1223 lutFilterNameString, fitBandString, notFitBandString):
1224 """
1225 Make the FGCM parameter catalog for persistence
1227 Parameters
1228 ----------
1229 parSchema: `lsst.afw.table.Schema`
1230 Parameter catalog schema
1231 pars: `numpy.ndarray`
1232 FGCM parameters to put into parCat
1233 parSuperStarFlat: `numpy.array`
1234 FGCM superstar flat array to put into parCat
1235 lutFilterNameString: `str`
1236 Combined string of all the lutFilterNames
1237 fitBandString: `str`
1238 Combined string of all the fitBands
1239 notFitBandString: `str`
1240 Combined string of all the bands not used in the fit
1242 Returns
1243 -------
1244 parCat: `afwTable.BasicCatalog`
1245 Atmosphere and instrumental model parameter catalog for persistence
1246 """
1248 parCat = afwTable.BaseCatalog(parSchema)
1249 parCat.reserve(1)
1251 # The parameter catalog just has one row, with many columns for all the
1252 # atmosphere and instrument fit parameters
1253 rec = parCat.addNew()
1255 # info section
1256 rec['nCcd'] = parInfo['NCCD']
1257 rec['lutFilterNames'] = lutFilterNameString
1258 rec['fitBands'] = fitBandString
1259 rec['notFitBands'] = notFitBandString
1260 # note these are not currently supported here.
1261 rec['hasExternalPwv'] = 0
1262 rec['hasExternalTau'] = 0
1264 # parameter section
1266 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1268 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1269 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1270 'parQeSysIntercept', 'compQeSysSlope',
1271 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1272 'parFilterOffset', 'parFilterOffsetFitFlag',
1273 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1274 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1275 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1276 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1277 'compModelErrSkyPivot', 'compModelErrPars',
1278 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1279 'compSigmaCal',
1280 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1281 'compRetrievedTauNight']
1283 for scalarName in scalarNames:
1284 rec[scalarName] = pars[scalarName.upper()]
1286 for arrName in arrNames:
1287 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1289 # superstar section
1290 rec['superstarSize'][:] = parSuperStarFlat.shape
1291 rec['superstar'][:] = parSuperStarFlat.flatten()
1293 return parCat
1295 def _makeFlagStarSchema(self):
1296 """
1297 Make the flagged-stars schema
1299 Returns
1300 -------
1301 flagStarSchema: `lsst.afw.table.Schema`
1302 """
1304 flagStarSchema = afwTable.Schema()
1306 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1307 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1309 return flagStarSchema
1311 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1312 """
1313 Make the flagged star catalog for persistence
1315 Parameters
1316 ----------
1317 flagStarSchema: `lsst.afw.table.Schema`
1318 Flagged star schema
1319 flagStarStruct: `numpy.ndarray`
1320 Flagged star structure from fgcm
1322 Returns
1323 -------
1324 flagStarCat: `lsst.afw.table.BaseCatalog`
1325 Flagged star catalog for persistence
1326 """
1328 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1329 flagStarCat.reserve(flagStarStruct.size)
1330 for i in range(flagStarStruct.size):
1331 flagStarCat.addNew()
1333 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1334 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1336 return flagStarCat