Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45import lsst.afw.table as afwTable
47from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
48from .utilities import extractReferenceMags
49from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
50from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51from .sedterms import SedboundarytermDict, SedtermDict
52from .utilities import lookupStaticCalibrations
54import fgcm
56__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument",),
61 defaultTemplates={"previousCycleNumber": "-1",
62 "cycleNumber": "0"}):
63 camera = connectionTypes.PrerequisiteInput(
64 doc="Camera instrument",
65 name="camera",
66 storageClass="Camera",
67 dimensions=("instrument",),
68 lookupFunction=lookupStaticCalibrations,
69 isCalibration=True,
70 )
72 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
73 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
74 "chromatic corrections."),
75 name="fgcmLookUpTable",
76 storageClass="Catalog",
77 dimensions=("instrument",),
78 deferLoad=True,
79 )
81 fgcmVisitCatalog = connectionTypes.PrerequisiteInput(
82 doc="Catalog of visit information for fgcm",
83 name="fgcmVisitCatalog",
84 storageClass="Catalog",
85 dimensions=("instrument",),
86 deferLoad=True,
87 )
89 fgcmStarObservations = connectionTypes.PrerequisiteInput(
90 doc="Catalog of star observations for fgcm",
91 name="fgcmStarObservations",
92 storageClass="Catalog",
93 dimensions=("instrument",),
94 deferLoad=True,
95 )
97 fgcmStarIds = connectionTypes.PrerequisiteInput(
98 doc="Catalog of fgcm calibration star IDs",
99 name="fgcmStarIds",
100 storageClass="Catalog",
101 dimensions=("instrument",),
102 deferLoad=True,
103 )
105 fgcmStarIndices = connectionTypes.PrerequisiteInput(
106 doc="Catalog of fgcm calibration star indices",
107 name="fgcmStarIndices",
108 storageClass="Catalog",
109 dimensions=("instrument",),
110 deferLoad=True,
111 )
113 fgcmReferenceStars = connectionTypes.PrerequisiteInput(
114 doc="Catalog of fgcm-matched reference stars",
115 name="fgcmReferenceStars",
116 storageClass="Catalog",
117 dimensions=("instrument",),
118 deferLoad=True,
119 )
121 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
122 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
123 name="fgcmFlaggedStars{previousCycleNumber}",
124 storageClass="Catalog",
125 dimensions=("instrument",),
126 deferLoad=True,
127 )
129 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
130 doc="Catalog of fgcm fit parameters from previous fit cycle",
131 name="fgcmFitParameters{previousCycleNumber}",
132 storageClass="Catalog",
133 dimensions=("instrument",),
134 deferLoad=True,
135 )
137 fgcmFitParameters = connectionTypes.Output(
138 doc="Catalog of fgcm fit parameters from current fit cycle",
139 name="fgcmFitParameters{cycleNumber}",
140 storageClass="Catalog",
141 dimensions=("instrument",),
142 )
144 fgcmFlaggedStars = connectionTypes.Output(
145 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
146 name="fgcmFlaggedStars{cycleNumber}",
147 storageClass="Catalog",
148 dimensions=("instrument",),
149 )
151 fgcmZeropoints = connectionTypes.Output(
152 doc="Catalog of fgcm zeropoint data from current fit cycle",
153 name="fgcmZeropoints{cycleNumber}",
154 storageClass="Catalog",
155 dimensions=("instrument",),
156 )
158 fgcmAtmosphereParameters = connectionTypes.Output(
159 doc="Catalog of atmospheric fit parameters from current fit cycle",
160 name="fgcmAtmosphereParameters{cycleNumber}",
161 storageClass="Catalog",
162 dimensions=("instrument",),
163 )
165 fgcmStandardStars = connectionTypes.Output(
166 doc="Catalog of standard star magnitudes from current fit cycle",
167 name="fgcmStandardStars{cycleNumber}",
168 storageClass="SimpleCatalog",
169 dimensions=("instrument",),
170 )
172 def __init__(self, *, config=None):
173 super().__init__(config=config)
175 if not config.doReferenceCalibration:
176 self.prerequisiteInputs.remove("fgcmReferenceStars")
178 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
179 raise ValueError("cycleNumber must be of integer format")
180 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
181 raise ValueError("previousCycleNumber must be of integer format")
182 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
183 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
185 if int(config.connections.cycleNumber) == 0:
186 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
187 self.prerequisiteInputs.remove("fgcmFitParametersInput")
189 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
190 self.outputs.remove("fgcmStandardStars")
192 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
193 self.outputs.remove("fgcmZeropoints")
194 self.outputs.remove("fgcmAtmosphereParameters")
197class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
198 pipelineConnections=FgcmFitCycleConnections):
199 """Config for FgcmFitCycle"""
201 bands = pexConfig.ListField(
202 doc="Bands to run calibration",
203 dtype=str,
204 default=[],
205 )
206 fitFlag = pexConfig.ListField(
207 doc=("Flag for which bands are directly constrained in the FGCM fit. "
208 "Bands set to 0 will have the atmosphere constrained from observations "
209 "in other bands on the same night. Must be same length as config.bands, "
210 "and matched band-by-band."),
211 dtype=int,
212 default=(0,),
213 optional=True,
214 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
215 "It will be removed after v20. Use fitBands instead."),
216 )
217 fitBands = pexConfig.ListField(
218 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
219 "the atmosphere constrained from the 'fitBands' on the same night. "
220 "Must be a subset of `config.bands`"),
221 dtype=str,
222 default=[],
223 )
224 requiredFlag = pexConfig.ListField(
225 doc=("Flag for which bands are required for a star to be considered a calibration "
226 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
227 "be same length as config.bands, and matched band-by-band."),
228 dtype=int,
229 default=(0,),
230 optional=True,
231 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
232 "It will be removed after v20. Use requiredBands instead."),
233 )
234 requiredBands = pexConfig.ListField(
235 doc=("Bands that are required for a star to be considered a calibration star. "
236 "Must be a subset of `config.bands`"),
237 dtype=str,
238 default=[],
239 )
240 filterMap = pexConfig.DictField(
241 doc="Mapping from 'filterName' to band.",
242 keytype=str,
243 itemtype=str,
244 default={},
245 deprecated=("This field is no longer used, and has been deprecated by "
246 "DM-28088. It will be removed after v22. Use "
247 "physicalFilterMap instead.")
248 )
249 # The following config will not be necessary after Gen2 retirement.
250 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
251 # is easiest to access in the config file.
252 physicalFilterMap = pexConfig.DictField(
253 doc="Mapping from 'physicalFilter' to band.",
254 keytype=str,
255 itemtype=str,
256 default={},
257 )
258 doReferenceCalibration = pexConfig.Field(
259 doc="Use reference catalog as additional constraint on calibration",
260 dtype=bool,
261 default=True,
262 )
263 refStarSnMin = pexConfig.Field(
264 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
265 dtype=float,
266 default=50.0,
267 )
268 refStarOutlierNSig = pexConfig.Field(
269 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
270 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
271 dtype=float,
272 default=4.0,
273 )
274 applyRefStarColorCuts = pexConfig.Field(
275 doc="Apply color cuts to reference stars?",
276 dtype=bool,
277 default=True,
278 )
279 nCore = pexConfig.Field(
280 doc="Number of cores to use",
281 dtype=int,
282 default=4,
283 )
284 nStarPerRun = pexConfig.Field(
285 doc="Number of stars to run in each chunk",
286 dtype=int,
287 default=200000,
288 )
289 nExpPerRun = pexConfig.Field(
290 doc="Number of exposures to run in each chunk",
291 dtype=int,
292 default=1000,
293 )
294 reserveFraction = pexConfig.Field(
295 doc="Fraction of stars to reserve for testing",
296 dtype=float,
297 default=0.1,
298 )
299 freezeStdAtmosphere = pexConfig.Field(
300 doc="Freeze atmosphere parameters to standard (for testing)",
301 dtype=bool,
302 default=False,
303 )
304 precomputeSuperStarInitialCycle = pexConfig.Field(
305 doc="Precompute superstar flat for initial cycle",
306 dtype=bool,
307 default=False,
308 )
309 superStarSubCcd = pexConfig.Field(
310 doc="Compute superstar flat on sub-ccd scale",
311 dtype=bool,
312 default=True,
313 optional=True,
314 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
315 "It will be removed after v20. Use superStarSubCcdDict instead."),
316 )
317 superStarSubCcdDict = pexConfig.DictField(
318 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
319 "Must have one entry per band."),
320 keytype=str,
321 itemtype=bool,
322 default={},
323 )
324 superStarSubCcdChebyshevOrder = pexConfig.Field(
325 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
326 "Global default is first-order polynomials, and should be overridden "
327 "on a camera-by-camera basis depending on the ISR."),
328 dtype=int,
329 default=1,
330 )
331 superStarSubCcdTriangular = pexConfig.Field(
332 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
333 "suppress high-order cross terms?"),
334 dtype=bool,
335 default=False,
336 )
337 superStarSigmaClip = pexConfig.Field(
338 doc="Number of sigma to clip outliers when selecting for superstar flats",
339 dtype=float,
340 default=5.0,
341 )
342 focalPlaneSigmaClip = pexConfig.Field(
343 doc="Number of sigma to clip outliers per focal-plane.",
344 dtype=float,
345 default=4.0,
346 )
347 ccdGraySubCcd = pexConfig.Field(
348 doc="Compute CCD gray terms on sub-ccd scale",
349 dtype=bool,
350 default=False,
351 optional=True,
352 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
353 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
354 )
355 ccdGraySubCcdDict = pexConfig.DictField(
356 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
357 "('ccd gray') on a sub-ccd scale."),
358 keytype=str,
359 itemtype=bool,
360 default={},
361 )
362 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
363 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
364 dtype=int,
365 default=1,
366 )
367 ccdGraySubCcdTriangular = pexConfig.Field(
368 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
369 "suppress high-order cross terms?"),
370 dtype=bool,
371 default=True,
372 )
373 ccdGrayFocalPlaneDict = pexConfig.DictField(
374 doc=("Per-band specification on whether to compute focal-plane residual "
375 "('ccd gray') corrections."),
376 keytype=str,
377 itemtype=bool,
378 default={},
379 )
380 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
381 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
382 "gray corrections. If there are fewer good CCDs then the gray "
383 "correction is computed per-ccd."),
384 dtype=int,
385 default=1,
386 )
387 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
388 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
389 dtype=int,
390 default=3,
391 )
392 cycleNumber = pexConfig.Field(
393 doc=("FGCM fit cycle number. This is automatically incremented after each run "
394 "and stage of outlier rejection. See cookbook for details."),
395 dtype=int,
396 default=None,
397 )
398 isFinalCycle = pexConfig.Field(
399 doc=("Is this the final cycle of the fitting? Will automatically compute final "
400 "selection of stars and photometric exposures, and will output zeropoints "
401 "and standard stars for use in fgcmOutputProducts"),
402 dtype=bool,
403 default=False,
404 )
405 maxIterBeforeFinalCycle = pexConfig.Field(
406 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
407 "will always be 0 in the final cycle for cleanup and final selection."),
408 dtype=int,
409 default=50,
410 )
411 deltaMagBkgOffsetPercentile = pexConfig.Field(
412 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
413 "offset from local background subtraction."),
414 dtype=float,
415 default=0.25,
416 )
417 deltaMagBkgPerCcd = pexConfig.Field(
418 doc=("Compute net offset from local background subtraction per-ccd? "
419 "Otherwise, use computation per visit."),
420 dtype=bool,
421 default=False,
422 )
423 utBoundary = pexConfig.Field(
424 doc="Boundary (in UTC) from day-to-day",
425 dtype=float,
426 default=None,
427 )
428 washMjds = pexConfig.ListField(
429 doc="Mirror wash MJDs",
430 dtype=float,
431 default=(0.0,),
432 )
433 epochMjds = pexConfig.ListField(
434 doc="Epoch boundaries in MJD",
435 dtype=float,
436 default=(0.0,),
437 )
438 minObsPerBand = pexConfig.Field(
439 doc="Minimum good observations per band",
440 dtype=int,
441 default=2,
442 )
443 # TODO: When DM-16511 is done, it will be possible to get the
444 # telescope latitude directly from the camera.
445 latitude = pexConfig.Field(
446 doc="Observatory latitude",
447 dtype=float,
448 default=None,
449 )
450 brightObsGrayMax = pexConfig.Field(
451 doc="Maximum gray extinction to be considered bright observation",
452 dtype=float,
453 default=0.15,
454 )
455 minStarPerCcd = pexConfig.Field(
456 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
457 "CCDs with fewer stars will have their calibration estimated from other "
458 "CCDs in the same visit, with zeropoint error increased accordingly."),
459 dtype=int,
460 default=5,
461 )
462 minCcdPerExp = pexConfig.Field(
463 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
464 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
465 dtype=int,
466 default=5,
467 )
468 maxCcdGrayErr = pexConfig.Field(
469 doc="Maximum error on CCD gray offset to be considered photometric",
470 dtype=float,
471 default=0.05,
472 )
473 minStarPerExp = pexConfig.Field(
474 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
475 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
476 dtype=int,
477 default=600,
478 )
479 minExpPerNight = pexConfig.Field(
480 doc="Minimum number of good exposures/visits to consider a partly photometric night",
481 dtype=int,
482 default=10,
483 )
484 expGrayInitialCut = pexConfig.Field(
485 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
486 "observations."),
487 dtype=float,
488 default=-0.25,
489 )
490 expGrayPhotometricCut = pexConfig.ListField(
491 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
492 "Must be same length as config.bands, and matched band-by-band."),
493 dtype=float,
494 default=(0.0,),
495 optional=True,
496 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
497 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
498 )
499 expGrayPhotometricCutDict = pexConfig.DictField(
500 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
501 "('gray term') for a visit to be considered photometric. Must have one "
502 "entry per band. Broad-band filters should be -0.05."),
503 keytype=str,
504 itemtype=float,
505 default={},
506 )
507 expGrayHighCut = pexConfig.ListField(
508 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
509 "Must be same length as config.bands, and matched band-by-band."),
510 dtype=float,
511 default=(0.0,),
512 optional=True,
513 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
514 "It will be removed after v20. Use expGrayHighCutDict instead."),
515 )
516 expGrayHighCutDict = pexConfig.DictField(
517 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
518 "('gray term') for a visit to be considered photometric. Must have one "
519 "entry per band. Broad-band filters should be 0.2."),
520 keytype=str,
521 itemtype=float,
522 default={},
523 )
524 expGrayRecoverCut = pexConfig.Field(
525 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
526 "Visits with more gray extinction will only get CCD zeropoints if there are "
527 "sufficient star observations (minStarPerCcd) on that CCD."),
528 dtype=float,
529 default=-1.0,
530 )
531 expVarGrayPhotometricCut = pexConfig.Field(
532 doc="Maximum exposure variance to be considered possibly photometric",
533 dtype=float,
534 default=0.0005,
535 optional=True,
536 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
537 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
538 )
539 expVarGrayPhotometricCutDict = pexConfig.DictField(
540 doc=("Per-band specification on maximum exposure variance to be considered possibly "
541 "photometric. Must have one entry per band. Broad-band filters should be "
542 "0.0005."),
543 keytype=str,
544 itemtype=float,
545 default={},
546 )
547 expGrayErrRecoverCut = pexConfig.Field(
548 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
549 "Visits with more gray variance will only get CCD zeropoints if there are "
550 "sufficient star observations (minStarPerCcd) on that CCD."),
551 dtype=float,
552 default=0.05,
553 )
554 aperCorrFitNBins = pexConfig.Field(
555 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
556 "no fit will be performed, and the config.aperCorrInputSlopes will be "
557 "used if available."),
558 dtype=int,
559 default=10,
560 )
561 aperCorrInputSlopes = pexConfig.ListField(
562 doc=("Aperture correction input slope parameters. These are used on the first "
563 "fit iteration, and aperture correction parameters will be updated from "
564 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
565 "when there is insufficient data to fit the parameters (e.g. tract mode). "
566 "If set, must be same length as config.bands, and matched band-by-band."),
567 dtype=float,
568 default=[],
569 optional=True,
570 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
571 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
572 )
573 aperCorrInputSlopeDict = pexConfig.DictField(
574 doc=("Per-band specification of aperture correction input slope parameters. These "
575 "are used on the first fit iteration, and aperture correction parameters will "
576 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
577 "to set this when there is insufficient data to fit the parameters (e.g. "
578 "tract mode)."),
579 keytype=str,
580 itemtype=float,
581 default={},
582 )
583 sedFudgeFactors = pexConfig.ListField(
584 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
585 "config.bands, and matched band-by-band."),
586 dtype=float,
587 default=(0,),
588 optional=True,
589 deprecated=("This field has been deprecated and will be removed after v20. "
590 "Please use sedSlopeTermMap and sedSlopeMap."),
591 )
592 sedboundaryterms = pexConfig.ConfigField(
593 doc="Mapping from bands to SED boundary term names used is sedterms.",
594 dtype=SedboundarytermDict,
595 )
596 sedterms = pexConfig.ConfigField(
597 doc="Mapping from terms to bands for fgcm linear SED approximations.",
598 dtype=SedtermDict,
599 )
600 sigFgcmMaxErr = pexConfig.Field(
601 doc="Maximum mag error for fitting sigma_FGCM",
602 dtype=float,
603 default=0.01,
604 )
605 sigFgcmMaxEGray = pexConfig.ListField(
606 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
607 "May be 1 element (same for all bands) or the same length as config.bands."),
608 dtype=float,
609 default=(0.05,),
610 optional=True,
611 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
612 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
613 )
614 sigFgcmMaxEGrayDict = pexConfig.DictField(
615 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
616 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
617 "should be 0.05."),
618 keytype=str,
619 itemtype=float,
620 default={},
621 )
622 ccdGrayMaxStarErr = pexConfig.Field(
623 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
624 "computation"),
625 dtype=float,
626 default=0.10,
627 )
628 approxThroughput = pexConfig.ListField(
629 doc=("Approximate overall throughput at start of calibration observations. "
630 "May be 1 element (same for all bands) or the same length as config.bands."),
631 dtype=float,
632 default=(1.0, ),
633 optional=True,
634 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
635 "It will be removed after v20. Use approxThroughputDict instead."),
636 )
637 approxThroughputDict = pexConfig.DictField(
638 doc=("Per-band specification of the approximate overall throughput at the start of "
639 "calibration observations. Must have one entry per band. Typically should "
640 "be 1.0."),
641 keytype=str,
642 itemtype=float,
643 default={},
644 )
645 sigmaCalRange = pexConfig.ListField(
646 doc="Allowed range for systematic error floor estimation",
647 dtype=float,
648 default=(0.001, 0.003),
649 )
650 sigmaCalFitPercentile = pexConfig.ListField(
651 doc="Magnitude percentile range to fit systematic error floor",
652 dtype=float,
653 default=(0.05, 0.15),
654 )
655 sigmaCalPlotPercentile = pexConfig.ListField(
656 doc="Magnitude percentile range to plot systematic error floor",
657 dtype=float,
658 default=(0.05, 0.95),
659 )
660 sigma0Phot = pexConfig.Field(
661 doc="Systematic error floor for all zeropoints",
662 dtype=float,
663 default=0.003,
664 )
665 mapLongitudeRef = pexConfig.Field(
666 doc="Reference longitude for plotting maps",
667 dtype=float,
668 default=0.0,
669 )
670 mapNSide = pexConfig.Field(
671 doc="Healpix nside for plotting maps",
672 dtype=int,
673 default=256,
674 )
675 outfileBase = pexConfig.Field(
676 doc="Filename start for plot output files",
677 dtype=str,
678 default=None,
679 )
680 starColorCuts = pexConfig.ListField(
681 doc="Encoded star-color cuts (to be cleaned up)",
682 dtype=str,
683 default=("NO_DATA",),
684 )
685 colorSplitIndices = pexConfig.ListField(
686 doc="Band indices to use to split stars by color",
687 dtype=int,
688 default=None,
689 optional=True,
690 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
691 "It will be removed after v20. Use colorSplitBands instead."),
692 )
693 colorSplitBands = pexConfig.ListField(
694 doc="Band names to use to split stars by color. Must have 2 entries.",
695 dtype=str,
696 length=2,
697 default=('g', 'i'),
698 )
699 modelMagErrors = pexConfig.Field(
700 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
701 dtype=bool,
702 default=True,
703 )
704 useQuadraticPwv = pexConfig.Field(
705 doc="Model PWV with a quadratic term for variation through the night?",
706 dtype=bool,
707 default=False,
708 )
709 instrumentParsPerBand = pexConfig.Field(
710 doc=("Model instrumental parameters per band? "
711 "Otherwise, instrumental parameters (QE changes with time) are "
712 "shared among all bands."),
713 dtype=bool,
714 default=False,
715 )
716 instrumentSlopeMinDeltaT = pexConfig.Field(
717 doc=("Minimum time change (in days) between observations to use in constraining "
718 "instrument slope."),
719 dtype=float,
720 default=20.0,
721 )
722 fitMirrorChromaticity = pexConfig.Field(
723 doc="Fit (intraband) mirror chromatic term?",
724 dtype=bool,
725 default=False,
726 )
727 coatingMjds = pexConfig.ListField(
728 doc="Mirror coating dates in MJD",
729 dtype=float,
730 default=(0.0,),
731 )
732 outputStandardsBeforeFinalCycle = pexConfig.Field(
733 doc="Output standard stars prior to final cycle? Used in debugging.",
734 dtype=bool,
735 default=False,
736 )
737 outputZeropointsBeforeFinalCycle = pexConfig.Field(
738 doc="Output standard stars prior to final cycle? Used in debugging.",
739 dtype=bool,
740 default=False,
741 )
742 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
743 doc=("Use star repeatability (instead of exposures) for computing photometric "
744 "cuts? Recommended for tract mode or bands with few exposures. "
745 "May be 1 element (same for all bands) or the same length as config.bands."),
746 dtype=bool,
747 default=(False,),
748 optional=True,
749 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
750 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
751 )
752 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
753 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
754 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
755 keytype=str,
756 itemtype=bool,
757 default={},
758 )
759 autoPhotometricCutNSig = pexConfig.Field(
760 doc=("Number of sigma for automatic computation of (low) photometric cut. "
761 "Cut is based on exposure gray width (per band), unless "
762 "useRepeatabilityForExpGrayCuts is set, in which case the star "
763 "repeatability is used (also per band)."),
764 dtype=float,
765 default=3.0,
766 )
767 autoHighCutNSig = pexConfig.Field(
768 doc=("Number of sigma for automatic computation of (high) outlier cut. "
769 "Cut is based on exposure gray width (per band), unless "
770 "useRepeatabilityForExpGrayCuts is set, in which case the star "
771 "repeatability is used (also per band)."),
772 dtype=float,
773 default=4.0,
774 )
775 quietMode = pexConfig.Field(
776 doc="Be less verbose with logging.",
777 dtype=bool,
778 default=False,
779 )
780 randomSeed = pexConfig.Field(
781 doc="Random seed for fgcm for consistency in tests.",
782 dtype=int,
783 default=None,
784 optional=True,
785 )
787 def validate(self):
788 super().validate()
790 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
791 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
792 raise RuntimeError(msg)
793 if self.connections.cycleNumber != str(self.cycleNumber):
794 msg = "cycleNumber in template must be equal to connections.cycleNumber"
795 raise RuntimeError(msg)
797 for band in self.fitBands:
798 if band not in self.bands:
799 msg = 'fitBand %s not in bands' % (band)
800 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
801 for band in self.requiredBands:
802 if band not in self.bands:
803 msg = 'requiredBand %s not in bands' % (band)
804 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
805 for band in self.colorSplitBands:
806 if band not in self.bands:
807 msg = 'colorSplitBand %s not in bands' % (band)
808 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
809 for band in self.bands:
810 if band not in self.superStarSubCcdDict:
811 msg = 'band %s not in superStarSubCcdDict' % (band)
812 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
813 self, msg)
814 if band not in self.ccdGraySubCcdDict:
815 msg = 'band %s not in ccdGraySubCcdDict' % (band)
816 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
817 self, msg)
818 if band not in self.expGrayPhotometricCutDict:
819 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
820 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
821 self, msg)
822 if band not in self.expGrayHighCutDict:
823 msg = 'band %s not in expGrayHighCutDict' % (band)
824 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
825 self, msg)
826 if band not in self.expVarGrayPhotometricCutDict:
827 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
828 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
829 self, msg)
830 if band not in self.sigFgcmMaxEGrayDict:
831 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
832 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
833 self, msg)
834 if band not in self.approxThroughputDict:
835 msg = 'band %s not in approxThroughputDict' % (band)
836 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
837 self, msg)
838 if band not in self.useRepeatabilityForExpGrayCutsDict:
839 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
840 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
841 self, msg)
844class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
845 """Subclass of TaskRunner for fgcmFitCycleTask
847 fgcmFitCycleTask.run() takes one argument, the butler, and uses
848 stars and visits previously extracted from dataRefs by
849 fgcmBuildStars.
850 This Runner does not perform any dataRef parallelization, but the FGCM
851 code called by the Task uses python multiprocessing (see the "ncores"
852 config option).
853 """
855 @staticmethod
856 def getTargetList(parsedCmd):
857 """
858 Return a list with one element, the butler.
859 """
860 return [parsedCmd.butler]
862 def __call__(self, butler):
863 """
864 Parameters
865 ----------
866 butler: `lsst.daf.persistence.Butler`
868 Returns
869 -------
870 exitStatus: `list` with `pipeBase.Struct`
871 exitStatus (0: success; 1: failure)
872 """
874 task = self.TaskClass(config=self.config, log=self.log)
876 exitStatus = 0
877 if self.doRaise:
878 task.runDataRef(butler)
879 else:
880 try:
881 task.runDataRef(butler)
882 except Exception as e:
883 exitStatus = 1
884 task.log.fatal("Failed: %s" % e)
885 if not isinstance(e, pipeBase.TaskError):
886 traceback.print_exc(file=sys.stderr)
888 task.writeMetadata(butler)
890 # The task does not return any results:
891 return [pipeBase.Struct(exitStatus=exitStatus)]
893 def run(self, parsedCmd):
894 """
895 Run the task, with no multiprocessing
897 Parameters
898 ----------
899 parsedCmd: ArgumentParser parsed command line
900 """
902 resultList = []
904 if self.precall(parsedCmd):
905 targetList = self.getTargetList(parsedCmd)
906 # make sure that we only get 1
907 resultList = self(targetList[0])
909 return resultList
912class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
913 """
914 Run Single fit cycle for FGCM global calibration
915 """
917 ConfigClass = FgcmFitCycleConfig
918 RunnerClass = FgcmFitCycleRunner
919 _DefaultName = "fgcmFitCycle"
921 def __init__(self, butler=None, initInputs=None, **kwargs):
922 super().__init__(**kwargs)
924 # no saving of metadata for now
925 def _getMetadataName(self):
926 return None
928 def runQuantum(self, butlerQC, inputRefs, outputRefs):
929 camera = butlerQC.get(inputRefs.camera)
931 dataRefDict = {}
933 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
934 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
935 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
936 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
937 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
938 if self.config.doReferenceCalibration:
939 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
940 if self.config.cycleNumber > 0:
941 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
942 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
944 fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
946 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
947 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
948 if self.outputZeropoints:
949 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
950 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
951 if self.outputStandards:
952 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
954 @pipeBase.timeMethod
955 def runDataRef(self, butler):
956 """
957 Run a single fit cycle for FGCM
959 Parameters
960 ----------
961 butler: `lsst.daf.persistence.Butler`
962 """
963 self._checkDatasetsExist(butler)
965 dataRefDict = {}
966 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
967 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
968 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
969 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
970 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
971 if self.config.doReferenceCalibration:
972 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
973 if self.config.cycleNumber > 0:
974 lastCycle = self.config.cycleNumber - 1
975 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
976 fgcmcycle=lastCycle)
977 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
978 fgcmcycle=lastCycle)
980 camera = butler.get('camera')
981 fgcmDatasetDict = self._fgcmFitCycle(camera, dataRefDict)
983 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
984 fgcmcycle=self.config.cycleNumber)
985 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
986 fgcmcycle=self.config.cycleNumber)
987 if self.outputZeropoints:
988 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
989 fgcmcycle=self.config.cycleNumber)
990 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
991 fgcmcycle=self.config.cycleNumber)
992 if self.outputStandards:
993 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
994 fgcmcycle=self.config.cycleNumber)
996 def writeConfig(self, butler, clobber=False, doBackup=True):
997 """Write the configuration used for processing the data, or check that an existing
998 one is equal to the new one if present. This is an override of the regular
999 version from pipe_base that knows about fgcmcycle.
1001 Parameters
1002 ----------
1003 butler : `lsst.daf.persistence.Butler`
1004 Data butler used to write the config. The config is written to dataset type
1005 `CmdLineTask._getConfigName`.
1006 clobber : `bool`, optional
1007 A boolean flag that controls what happens if a config already has been saved:
1008 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1009 - `False`: raise `TaskError` if this config does not match the existing config.
1010 doBackup : `bool`, optional
1011 Set to `True` to backup the config files if clobbering.
1012 """
1013 configName = self._getConfigName()
1014 if configName is None:
1015 return
1016 if clobber:
1017 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1018 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1019 # this may be subject to a race condition; see #2789
1020 try:
1021 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1022 except Exception as exc:
1023 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1024 (configName, exc))
1026 def logConfigMismatch(msg):
1027 self.log.fatal("Comparing configuration: %s", msg)
1029 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1030 raise pipeBase.TaskError(
1031 f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1032 " must be consistent within the same output repo (override with --clobber-config)")
1033 else:
1034 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1036 def _fgcmFitCycle(self, camera, dataRefDict):
1037 """
1038 Run the fit cycle
1040 Parameters
1041 ----------
1042 camera : `lsst.afw.cameraGeom.Camera`
1043 dataRefDict : `dict`
1044 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1045 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1046 dataRef dictionary with keys:
1048 ``"fgcmLookUpTable"``
1049 dataRef for the FGCM look-up table.
1050 ``"fgcmVisitCatalog"``
1051 dataRef for visit summary catalog.
1052 ``"fgcmStarObservations"``
1053 dataRef for star observation catalog.
1054 ``"fgcmStarIds"``
1055 dataRef for star id catalog.
1056 ``"fgcmStarIndices"``
1057 dataRef for star index catalog.
1058 ``"fgcmReferenceStars"``
1059 dataRef for matched reference star catalog.
1060 ``"fgcmFlaggedStars"``
1061 dataRef for flagged star catalog.
1062 ``"fgcmFitParameters"``
1063 dataRef for fit parameter catalog.
1065 Returns
1066 -------
1067 fgcmDatasetDict : `dict`
1068 Dictionary of datasets to persist.
1069 """
1070 # Set defaults on whether to output standards and zeropoints
1071 self.maxIter = self.config.maxIterBeforeFinalCycle
1072 self.outputStandards = self.config.outputStandardsBeforeFinalCycle
1073 self.outputZeropoints = self.config.outputZeropointsBeforeFinalCycle
1074 self.resetFitParameters = True
1076 if self.config.isFinalCycle:
1077 # This is the final fit cycle, so we do not want to reset fit
1078 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1079 # and we always want to output standards and zeropoints
1080 self.maxIter = 0
1081 self.outputStandards = True
1082 self.outputZeropoints = True
1083 self.resetFitParameters = False
1085 lutCat = dataRefDict['fgcmLookUpTable'].get()
1086 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1087 dict(self.config.physicalFilterMap))
1088 del lutCat
1090 configDict = makeConfigDict(self.config, self.log, camera,
1091 self.maxIter, self.resetFitParameters,
1092 self.outputZeropoints,
1093 lutIndexVals[0]['FILTERNAMES'])
1095 # next we need the exposure/visit information
1096 visitCat = dataRefDict['fgcmVisitCatalog'].get()
1097 fgcmExpInfo = translateVisitCatalog(visitCat)
1098 del visitCat
1100 # Use the first orientation.
1101 # TODO: DM-21215 will generalize to arbitrary camera orientations
1102 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1104 noFitsDict = {'lutIndex': lutIndexVals,
1105 'lutStd': lutStd,
1106 'expInfo': fgcmExpInfo,
1107 'ccdOffsets': ccdOffsets}
1109 # set up the fitter object
1110 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1111 noFitsDict=noFitsDict, noOutput=True)
1113 # create the parameter object
1114 if (fgcmFitCycle.initialCycle):
1115 # cycle = 0, initial cycle
1116 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1117 fgcmLut,
1118 fgcmExpInfo)
1119 else:
1120 inParInfo, inParams, inSuperStar = self._loadParameters(dataRefDict['fgcmFitParameters'])
1121 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1122 fgcmExpInfo,
1123 inParInfo,
1124 inParams,
1125 inSuperStar)
1127 # set up the stars...
1128 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1130 starObs = dataRefDict['fgcmStarObservations'].get()
1131 starIds = dataRefDict['fgcmStarIds'].get()
1132 starIndices = dataRefDict['fgcmStarIndices'].get()
1134 # grab the flagged stars if available
1135 if 'fgcmFlaggedStars' in dataRefDict:
1136 flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1137 flagId = flaggedStars['objId'][:]
1138 flagFlag = flaggedStars['objFlag'][:]
1139 else:
1140 flaggedStars = None
1141 flagId = None
1142 flagFlag = None
1144 if self.config.doReferenceCalibration:
1145 refStars = dataRefDict['fgcmReferenceStars'].get()
1147 refMag, refMagErr = extractReferenceMags(refStars,
1148 self.config.bands,
1149 self.config.physicalFilterMap)
1150 refId = refStars['fgcm_id'][:]
1151 else:
1152 refStars = None
1153 refId = None
1154 refMag = None
1155 refMagErr = None
1157 # match star observations to visits
1158 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1159 # actually be transferred into fgcm using the indexing below.
1160 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1162 # The fgcmStars.loadStars method will copy all the star information into
1163 # special shared memory objects that will not blow up the memory usage when
1164 # used with python multiprocessing. Once all the numbers are copied,
1165 # it is necessary to release all references to the objects that previously
1166 # stored the data to ensure that the garbage collector can clear the memory,
1167 # and ensure that this memory is not copied when multiprocessing kicks in.
1169 # We determine the conversion from the native units (typically radians) to
1170 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1171 # numpy arrays rather than Angles, which would we approximately 600x slower.
1172 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1174 fgcmStars.loadStars(fgcmPars,
1175 starObs['visit'][starIndices['obsIndex']],
1176 starObs['ccd'][starIndices['obsIndex']],
1177 starObs['ra'][starIndices['obsIndex']] * conv,
1178 starObs['dec'][starIndices['obsIndex']] * conv,
1179 starObs['instMag'][starIndices['obsIndex']],
1180 starObs['instMagErr'][starIndices['obsIndex']],
1181 fgcmExpInfo['FILTERNAME'][visitIndex],
1182 starIds['fgcm_id'][:],
1183 starIds['ra'][:],
1184 starIds['dec'][:],
1185 starIds['obsArrIndex'][:],
1186 starIds['nObs'][:],
1187 obsX=starObs['x'][starIndices['obsIndex']],
1188 obsY=starObs['y'][starIndices['obsIndex']],
1189 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1190 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1191 refID=refId,
1192 refMag=refMag,
1193 refMagErr=refMagErr,
1194 flagID=flagId,
1195 flagFlag=flagFlag,
1196 computeNobs=True)
1198 # Release all references to temporary objects holding star data (see above)
1199 del starObs
1200 del starIds
1201 del starIndices
1202 del flagId
1203 del flagFlag
1204 del flaggedStars
1205 del refStars
1206 del refId
1207 del refMag
1208 del refMagErr
1210 # and set the bits in the cycle object
1211 fgcmFitCycle.setLUT(fgcmLut)
1212 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1213 fgcmFitCycle.setPars(fgcmPars)
1215 # finish the setup
1216 fgcmFitCycle.finishSetup()
1218 # and run
1219 fgcmFitCycle.run()
1221 ##################
1222 # Persistance
1223 ##################
1225 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1227 # Output the config for the next cycle
1228 # We need to make a copy since the input one has been frozen
1230 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1231 i, b in enumerate(self.config.bands)}
1232 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1233 i, band in enumerate(self.config.bands)}
1235 outConfig = copy.copy(self.config)
1236 outConfig.update(cycleNumber=(self.config.cycleNumber + 1),
1237 precomputeSuperStarInitialCycle=False,
1238 freezeStdAtmosphere=False,
1239 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1240 expGrayHighCutDict=updatedHighCutDict)
1242 outConfig.connections.update(previousCycleNumber=str(self.config.cycleNumber),
1243 cycleNumber=str(self.config.cycleNumber + 1))
1245 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1246 outConfig.cycleNumber)
1247 outConfig.save(configFileName)
1249 if self.config.isFinalCycle == 1:
1250 # We are done, ready to output products
1251 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1252 else:
1253 self.log.info("Saved config for next cycle to %s" % (configFileName))
1254 self.log.info("Be sure to look at:")
1255 self.log.info(" config.expGrayPhotometricCut")
1256 self.log.info(" config.expGrayHighCut")
1257 self.log.info("If you are satisfied with the fit, please set:")
1258 self.log.info(" config.isFinalCycle = True")
1260 return fgcmDatasetDict
1262 def _checkDatasetsExist(self, butler):
1263 """
1264 Check if necessary datasets exist to run fgcmFitCycle
1266 Parameters
1267 ----------
1268 butler: `lsst.daf.persistence.Butler`
1270 Raises
1271 ------
1272 RuntimeError
1273 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1274 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1275 If cycleNumber > 0, then also checks for fgcmFitParameters,
1276 fgcmFlaggedStars.
1277 """
1279 if not butler.datasetExists('fgcmVisitCatalog'):
1280 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1281 if not butler.datasetExists('fgcmStarObservations'):
1282 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1283 if not butler.datasetExists('fgcmStarIds'):
1284 raise RuntimeError("Could not find fgcmStarIds in repo!")
1285 if not butler.datasetExists('fgcmStarIndices'):
1286 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1287 if not butler.datasetExists('fgcmLookUpTable'):
1288 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1290 # Need additional datasets if we are not the initial cycle
1291 if (self.config.cycleNumber > 0):
1292 if not butler.datasetExists('fgcmFitParameters',
1293 fgcmcycle=self.config.cycleNumber-1):
1294 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1295 (self.config.cycleNumber-1))
1296 if not butler.datasetExists('fgcmFlaggedStars',
1297 fgcmcycle=self.config.cycleNumber-1):
1298 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1299 (self.config.cycleNumber-1))
1301 # And additional dataset if we want reference calibration
1302 if self.config.doReferenceCalibration:
1303 if not butler.datasetExists('fgcmReferenceStars'):
1304 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1305 "doReferenceCalibration is True.")
1307 def _loadParameters(self, parDataRef):
1308 """
1309 Load FGCM parameters from a previous fit cycle
1311 Parameters
1312 ----------
1313 parDataRef : `lsst.daf.persistence.ButlerDataRef` or `lsst.daf.butler.DeferredDatasetHandle`
1314 dataRef for previous fit parameter catalog.
1316 Returns
1317 -------
1318 inParInfo: `numpy.ndarray`
1319 Numpy array parameter information formatted for input to fgcm
1320 inParameters: `numpy.ndarray`
1321 Numpy array parameter values formatted for input to fgcm
1322 inSuperStar: `numpy.array`
1323 Superstar flat formatted for input to fgcm
1324 """
1325 # note that we already checked that this is available
1326 parCat = parDataRef.get()
1328 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1329 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1331 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1332 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1333 (parLutFilterNames.size, )),
1334 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1335 ('LNTAUUNIT', 'f8'),
1336 ('LNTAUSLOPEUNIT', 'f8'),
1337 ('ALPHAUNIT', 'f8'),
1338 ('LNPWVUNIT', 'f8'),
1339 ('LNPWVSLOPEUNIT', 'f8'),
1340 ('LNPWVQUADRATICUNIT', 'f8'),
1341 ('LNPWVGLOBALUNIT', 'f8'),
1342 ('O3UNIT', 'f8'),
1343 ('QESYSUNIT', 'f8'),
1344 ('FILTEROFFSETUNIT', 'f8'),
1345 ('HASEXTERNALPWV', 'i2'),
1346 ('HASEXTERNALTAU', 'i2')])
1347 inParInfo['NCCD'] = parCat['nCcd']
1348 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1349 inParInfo['FITBANDS'][:] = parFitBands
1350 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1351 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1353 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1354 ('PARO3', 'f8', (parCat['parO3'].size, )),
1355 ('PARLNTAUINTERCEPT', 'f8',
1356 (parCat['parLnTauIntercept'].size, )),
1357 ('PARLNTAUSLOPE', 'f8',
1358 (parCat['parLnTauSlope'].size, )),
1359 ('PARLNPWVINTERCEPT', 'f8',
1360 (parCat['parLnPwvIntercept'].size, )),
1361 ('PARLNPWVSLOPE', 'f8',
1362 (parCat['parLnPwvSlope'].size, )),
1363 ('PARLNPWVQUADRATIC', 'f8',
1364 (parCat['parLnPwvQuadratic'].size, )),
1365 ('PARQESYSINTERCEPT', 'f8',
1366 (parCat['parQeSysIntercept'].size, )),
1367 ('COMPQESYSSLOPE', 'f8',
1368 (parCat['compQeSysSlope'].size, )),
1369 ('PARFILTEROFFSET', 'f8',
1370 (parCat['parFilterOffset'].size, )),
1371 ('PARFILTEROFFSETFITFLAG', 'i2',
1372 (parCat['parFilterOffsetFitFlag'].size, )),
1373 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1374 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1375 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1376 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1377 ('COMPABSTHROUGHPUT', 'f8',
1378 (parCat['compAbsThroughput'].size, )),
1379 ('COMPREFOFFSET', 'f8',
1380 (parCat['compRefOffset'].size, )),
1381 ('COMPREFSIGMA', 'f8',
1382 (parCat['compRefSigma'].size, )),
1383 ('COMPMIRRORCHROMATICITY', 'f8',
1384 (parCat['compMirrorChromaticity'].size, )),
1385 ('MIRRORCHROMATICITYPIVOT', 'f8',
1386 (parCat['mirrorChromaticityPivot'].size, )),
1387 ('COMPMEDIANSEDSLOPE', 'f8',
1388 (parCat['compMedianSedSlope'].size, )),
1389 ('COMPAPERCORRPIVOT', 'f8',
1390 (parCat['compAperCorrPivot'].size, )),
1391 ('COMPAPERCORRSLOPE', 'f8',
1392 (parCat['compAperCorrSlope'].size, )),
1393 ('COMPAPERCORRSLOPEERR', 'f8',
1394 (parCat['compAperCorrSlopeErr'].size, )),
1395 ('COMPAPERCORRRANGE', 'f8',
1396 (parCat['compAperCorrRange'].size, )),
1397 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1398 (parCat['compModelErrExptimePivot'].size, )),
1399 ('COMPMODELERRFWHMPIVOT', 'f8',
1400 (parCat['compModelErrFwhmPivot'].size, )),
1401 ('COMPMODELERRSKYPIVOT', 'f8',
1402 (parCat['compModelErrSkyPivot'].size, )),
1403 ('COMPMODELERRPARS', 'f8',
1404 (parCat['compModelErrPars'].size, )),
1405 ('COMPEXPGRAY', 'f8',
1406 (parCat['compExpGray'].size, )),
1407 ('COMPVARGRAY', 'f8',
1408 (parCat['compVarGray'].size, )),
1409 ('COMPEXPDELTAMAGBKG', 'f8',
1410 (parCat['compExpDeltaMagBkg'].size, )),
1411 ('COMPNGOODSTARPEREXP', 'i4',
1412 (parCat['compNGoodStarPerExp'].size, )),
1413 ('COMPSIGFGCM', 'f8',
1414 (parCat['compSigFgcm'].size, )),
1415 ('COMPSIGMACAL', 'f8',
1416 (parCat['compSigmaCal'].size, )),
1417 ('COMPRETRIEVEDLNPWV', 'f8',
1418 (parCat['compRetrievedLnPwv'].size, )),
1419 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1420 (parCat['compRetrievedLnPwvRaw'].size, )),
1421 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1422 (parCat['compRetrievedLnPwvFlag'].size, )),
1423 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1424 (parCat['compRetrievedTauNight'].size, ))])
1426 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1427 inParams['PARO3'][:] = parCat['parO3'][0, :]
1428 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1429 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1430 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1431 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1432 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1433 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1434 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1435 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1436 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1437 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1438 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1439 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1440 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1441 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1442 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1443 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1444 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1445 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1446 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1447 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1448 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1449 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1450 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1451 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1452 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1453 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1454 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1455 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1456 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1457 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1458 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1459 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1460 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1461 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1462 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1463 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1465 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1466 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1468 return (inParInfo, inParams, inSuperStar)
1470 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1471 """
1472 Persist FGCM datasets through the butler.
1474 Parameters
1475 ----------
1476 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1477 Fgcm Fit cycle object
1478 """
1479 fgcmDatasetDict = {}
1481 # Save the parameters
1482 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1484 parSchema = afwTable.Schema()
1486 comma = ','
1487 lutFilterNameString = comma.join([n.decode('utf-8')
1488 for n in parInfo['LUTFILTERNAMES'][0]])
1489 fitBandString = comma.join([n.decode('utf-8')
1490 for n in parInfo['FITBANDS'][0]])
1492 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1493 lutFilterNameString, fitBandString)
1494 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1495 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1496 lutFilterNameString, fitBandString)
1498 fgcmDatasetDict['fgcmFitParameters'] = parCat
1500 # Save the indices of the flagged stars
1501 # (stars that have been (a) reserved from the fit for testing and
1502 # (b) bad stars that have failed quality checks.)
1503 flagStarSchema = self._makeFlagStarSchema()
1504 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1505 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1507 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1509 # Save the zeropoint information and atmospheres only if desired
1510 if self.outputZeropoints:
1511 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1512 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1514 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1515 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1517 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1519 # Save atmosphere values
1520 # These are generated by the same code that generates zeropoints
1521 atmSchema = makeAtmSchema()
1522 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1524 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1526 # Save the standard stars (if configured)
1527 if self.outputStandards:
1528 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1529 stdSchema = makeStdSchema(len(goodBands))
1530 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1532 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1534 return fgcmDatasetDict
1536 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1537 lutFilterNameString, fitBandString):
1538 """
1539 Make the parameter persistence schema
1541 Parameters
1542 ----------
1543 parInfo: `numpy.ndarray`
1544 Parameter information returned by fgcm
1545 pars: `numpy.ndarray`
1546 Parameter values returned by fgcm
1547 parSuperStarFlat: `numpy.array`
1548 Superstar flat values returned by fgcm
1549 lutFilterNameString: `str`
1550 Combined string of all the lutFilterNames
1551 fitBandString: `str`
1552 Combined string of all the fitBands
1554 Returns
1555 -------
1556 parSchema: `afwTable.schema`
1557 """
1559 parSchema = afwTable.Schema()
1561 # parameter info section
1562 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1563 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1564 size=len(lutFilterNameString))
1565 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1566 size=len(fitBandString))
1567 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1568 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1569 doc='Step units for ln(AOD) slope')
1570 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1571 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1572 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1573 doc='Step units for ln(pwv) slope')
1574 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1575 doc='Step units for ln(pwv) quadratic term')
1576 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1577 doc='Step units for global ln(pwv) parameters')
1578 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1579 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1580 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1581 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1582 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1584 # parameter section
1585 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1586 size=pars['PARALPHA'].size)
1587 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1588 size=pars['PARO3'].size)
1589 parSchema.addField('parLnTauIntercept', type='ArrayD',
1590 doc='ln(Tau) intercept parameter vector',
1591 size=pars['PARLNTAUINTERCEPT'].size)
1592 parSchema.addField('parLnTauSlope', type='ArrayD',
1593 doc='ln(Tau) slope parameter vector',
1594 size=pars['PARLNTAUSLOPE'].size)
1595 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1596 size=pars['PARLNPWVINTERCEPT'].size)
1597 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1598 size=pars['PARLNPWVSLOPE'].size)
1599 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1600 size=pars['PARLNPWVQUADRATIC'].size)
1601 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1602 size=pars['PARQESYSINTERCEPT'].size)
1603 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1604 size=pars[0]['COMPQESYSSLOPE'].size)
1605 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1606 size=pars['PARFILTEROFFSET'].size)
1607 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1608 size=pars['PARFILTEROFFSETFITFLAG'].size)
1609 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1610 doc='Global scale for retrieved ln(pwv)')
1611 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1612 doc='Global offset for retrieved ln(pwv)')
1613 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1614 doc='Nightly offset for retrieved ln(pwv)',
1615 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1616 parSchema.addField('compAbsThroughput', type='ArrayD',
1617 doc='Absolute throughput (relative to transmission curves)',
1618 size=pars['COMPABSTHROUGHPUT'].size)
1619 parSchema.addField('compRefOffset', type='ArrayD',
1620 doc='Offset between reference stars and calibrated stars',
1621 size=pars['COMPREFOFFSET'].size)
1622 parSchema.addField('compRefSigma', type='ArrayD',
1623 doc='Width of reference star/calibrated star distribution',
1624 size=pars['COMPREFSIGMA'].size)
1625 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1626 doc='Computed mirror chromaticity terms',
1627 size=pars['COMPMIRRORCHROMATICITY'].size)
1628 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1629 doc='Mirror chromaticity pivot mjd',
1630 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1631 parSchema.addField('compMedianSedSlope', type='ArrayD',
1632 doc='Computed median SED slope (per band)',
1633 size=pars['COMPMEDIANSEDSLOPE'].size)
1634 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1635 size=pars['COMPAPERCORRPIVOT'].size)
1636 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1637 size=pars['COMPAPERCORRSLOPE'].size)
1638 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1639 size=pars['COMPAPERCORRSLOPEERR'].size)
1640 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1641 size=pars['COMPAPERCORRRANGE'].size)
1642 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1643 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1644 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1645 size=pars['COMPMODELERRFWHMPIVOT'].size)
1646 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1647 size=pars['COMPMODELERRSKYPIVOT'].size)
1648 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1649 size=pars['COMPMODELERRPARS'].size)
1650 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1651 size=pars['COMPEXPGRAY'].size)
1652 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1653 size=pars['COMPVARGRAY'].size)
1654 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1655 doc='Computed exposure offset due to background',
1656 size=pars['COMPEXPDELTAMAGBKG'].size)
1657 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1658 doc='Computed number of good stars per exposure',
1659 size=pars['COMPNGOODSTARPEREXP'].size)
1660 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1661 size=pars['COMPSIGFGCM'].size)
1662 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1663 size=pars['COMPSIGMACAL'].size)
1664 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1665 size=pars['COMPRETRIEVEDLNPWV'].size)
1666 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1667 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1668 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1669 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1670 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1671 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1672 # superstarflat section
1673 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1674 size=4)
1675 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1676 size=parSuperStarFlat.size)
1678 return parSchema
1680 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1681 lutFilterNameString, fitBandString):
1682 """
1683 Make the FGCM parameter catalog for persistence
1685 Parameters
1686 ----------
1687 parSchema: `lsst.afw.table.Schema`
1688 Parameter catalog schema
1689 pars: `numpy.ndarray`
1690 FGCM parameters to put into parCat
1691 parSuperStarFlat: `numpy.array`
1692 FGCM superstar flat array to put into parCat
1693 lutFilterNameString: `str`
1694 Combined string of all the lutFilterNames
1695 fitBandString: `str`
1696 Combined string of all the fitBands
1698 Returns
1699 -------
1700 parCat: `afwTable.BasicCatalog`
1701 Atmosphere and instrumental model parameter catalog for persistence
1702 """
1704 parCat = afwTable.BaseCatalog(parSchema)
1705 parCat.reserve(1)
1707 # The parameter catalog just has one row, with many columns for all the
1708 # atmosphere and instrument fit parameters
1709 rec = parCat.addNew()
1711 # info section
1712 rec['nCcd'] = parInfo['NCCD']
1713 rec['lutFilterNames'] = lutFilterNameString
1714 rec['fitBands'] = fitBandString
1715 # note these are not currently supported here.
1716 rec['hasExternalPwv'] = 0
1717 rec['hasExternalTau'] = 0
1719 # parameter section
1721 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1723 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1724 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1725 'parQeSysIntercept', 'compQeSysSlope',
1726 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1727 'parFilterOffset', 'parFilterOffsetFitFlag',
1728 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1729 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1730 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1731 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1732 'compModelErrSkyPivot', 'compModelErrPars',
1733 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1734 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1735 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1736 'compRetrievedTauNight']
1738 for scalarName in scalarNames:
1739 rec[scalarName] = pars[scalarName.upper()]
1741 for arrName in arrNames:
1742 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1744 # superstar section
1745 rec['superstarSize'][:] = parSuperStarFlat.shape
1746 rec['superstar'][:] = parSuperStarFlat.flatten()
1748 return parCat
1750 def _makeFlagStarSchema(self):
1751 """
1752 Make the flagged-stars schema
1754 Returns
1755 -------
1756 flagStarSchema: `lsst.afw.table.Schema`
1757 """
1759 flagStarSchema = afwTable.Schema()
1761 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1762 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1764 return flagStarSchema
1766 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1767 """
1768 Make the flagged star catalog for persistence
1770 Parameters
1771 ----------
1772 flagStarSchema: `lsst.afw.table.Schema`
1773 Flagged star schema
1774 flagStarStruct: `numpy.ndarray`
1775 Flagged star structure from fgcm
1777 Returns
1778 -------
1779 flagStarCat: `lsst.afw.table.BaseCatalog`
1780 Flagged star catalog for persistence
1781 """
1783 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1784 flagStarCat.resize(flagStarStruct.size)
1786 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1787 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1789 return flagStarCat