Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45import lsst.afw.table as afwTable
46from lsst.utils.timer import timeMethod
48from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
49from .utilities import extractReferenceMags
50from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
51from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
52from .sedterms import SedboundarytermDict, SedtermDict
53from .utilities import lookupStaticCalibrations
55import fgcm
57__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
59MULTIPLE_CYCLES_MAX = 10
62class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
63 dimensions=("instrument",),
64 defaultTemplates={"previousCycleNumber": "-1",
65 "cycleNumber": "0"}):
66 camera = connectionTypes.PrerequisiteInput(
67 doc="Camera instrument",
68 name="camera",
69 storageClass="Camera",
70 dimensions=("instrument",),
71 lookupFunction=lookupStaticCalibrations,
72 isCalibration=True,
73 )
75 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
76 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
77 "chromatic corrections."),
78 name="fgcmLookUpTable",
79 storageClass="Catalog",
80 dimensions=("instrument",),
81 deferLoad=True,
82 )
84 fgcmVisitCatalog = connectionTypes.Input(
85 doc="Catalog of visit information for fgcm",
86 name="fgcmVisitCatalog",
87 storageClass="Catalog",
88 dimensions=("instrument",),
89 deferLoad=True,
90 )
92 fgcmStarObservations = connectionTypes.Input(
93 doc="Catalog of star observations for fgcm",
94 name="fgcmStarObservations",
95 storageClass="Catalog",
96 dimensions=("instrument",),
97 deferLoad=True,
98 )
100 fgcmStarIds = connectionTypes.Input(
101 doc="Catalog of fgcm calibration star IDs",
102 name="fgcmStarIds",
103 storageClass="Catalog",
104 dimensions=("instrument",),
105 deferLoad=True,
106 )
108 fgcmStarIndices = connectionTypes.Input(
109 doc="Catalog of fgcm calibration star indices",
110 name="fgcmStarIndices",
111 storageClass="Catalog",
112 dimensions=("instrument",),
113 deferLoad=True,
114 )
116 fgcmReferenceStars = connectionTypes.Input(
117 doc="Catalog of fgcm-matched reference stars",
118 name="fgcmReferenceStars",
119 storageClass="Catalog",
120 dimensions=("instrument",),
121 deferLoad=True,
122 )
124 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
125 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
126 name="fgcmFlaggedStars{previousCycleNumber}",
127 storageClass="Catalog",
128 dimensions=("instrument",),
129 deferLoad=True,
130 )
132 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
133 doc="Catalog of fgcm fit parameters from previous fit cycle",
134 name="fgcmFitParameters{previousCycleNumber}",
135 storageClass="Catalog",
136 dimensions=("instrument",),
137 deferLoad=True,
138 )
140 fgcmFitParameters = connectionTypes.Output(
141 doc="Catalog of fgcm fit parameters from current fit cycle",
142 name="fgcmFitParameters{cycleNumber}",
143 storageClass="Catalog",
144 dimensions=("instrument",),
145 )
147 fgcmFlaggedStars = connectionTypes.Output(
148 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
149 name="fgcmFlaggedStars{cycleNumber}",
150 storageClass="Catalog",
151 dimensions=("instrument",),
152 )
154 fgcmZeropoints = connectionTypes.Output(
155 doc="Catalog of fgcm zeropoint data from current fit cycle",
156 name="fgcmZeropoints{cycleNumber}",
157 storageClass="Catalog",
158 dimensions=("instrument",),
159 )
161 fgcmAtmosphereParameters = connectionTypes.Output(
162 doc="Catalog of atmospheric fit parameters from current fit cycle",
163 name="fgcmAtmosphereParameters{cycleNumber}",
164 storageClass="Catalog",
165 dimensions=("instrument",),
166 )
168 fgcmStandardStars = connectionTypes.Output(
169 doc="Catalog of standard star magnitudes from current fit cycle",
170 name="fgcmStandardStars{cycleNumber}",
171 storageClass="SimpleCatalog",
172 dimensions=("instrument",),
173 )
175 # Add connections for running multiple cycles
176 # This uses vars() to alter the class __dict__ to programmatically
177 # write many similar outputs.
178 for cycle in range(MULTIPLE_CYCLES_MAX):
179 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
180 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
181 name=f"fgcmFitParameters{cycle}",
182 storageClass="Catalog",
183 dimensions=("instrument",),
184 )
185 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
186 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
187 name=f"fgcmFlaggedStars{cycle}",
188 storageClass="Catalog",
189 dimensions=("instrument",),
190 )
191 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
192 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
193 name=f"fgcmZeropoints{cycle}",
194 storageClass="Catalog",
195 dimensions=("instrument",),
196 )
197 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
198 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
199 name=f"fgcmAtmosphereParameters{cycle}",
200 storageClass="Catalog",
201 dimensions=("instrument",),
202 )
203 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
204 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
205 name=f"fgcmStandardStars{cycle}",
206 storageClass="SimpleCatalog",
207 dimensions=("instrument",),
208 )
210 def __init__(self, *, config=None):
211 super().__init__(config=config)
213 if not config.doReferenceCalibration:
214 self.inputs.remove("fgcmReferenceStars")
216 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
217 raise ValueError("cycleNumber must be of integer format")
218 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
219 raise ValueError("previousCycleNumber must be of integer format")
220 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
221 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
223 if int(config.connections.cycleNumber) == 0:
224 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
225 self.prerequisiteInputs.remove("fgcmFitParametersInput")
227 if not self.config.doMultipleCycles:
228 # Single-cycle run
229 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
230 self.outputs.remove("fgcmStandardStars")
232 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
233 self.outputs.remove("fgcmZeropoints")
234 self.outputs.remove("fgcmAtmosphereParameters")
236 # Remove all multiple cycle outputs
237 for cycle in range(0, MULTIPLE_CYCLES_MAX):
238 self.outputs.remove(f"fgcmFitParameters{cycle}")
239 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
240 self.outputs.remove(f"fgcmZeropoints{cycle}")
241 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
242 self.outputs.remove(f"fgcmStandardStars{cycle}")
244 else:
245 # Multiple-cycle run
246 # Remove single-cycle outputs
247 self.outputs.remove("fgcmFitParameters")
248 self.outputs.remove("fgcmFlaggedStars")
249 self.outputs.remove("fgcmZeropoints")
250 self.outputs.remove("fgcmAtmosphereParameters")
251 self.outputs.remove("fgcmStandardStars")
253 # Remove outputs from cycles that are not used
254 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
255 MULTIPLE_CYCLES_MAX):
256 self.outputs.remove(f"fgcmFitParameters{cycle}")
257 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
258 self.outputs.remove(f"fgcmZeropoints{cycle}")
259 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
260 self.outputs.remove(f"fgcmStandardStars{cycle}")
262 # Remove non-final-cycle outputs if necessary
263 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
264 if not self.config.outputZeropointsBeforeFinalCycle:
265 self.outputs.remove(f"fgcmZeropoints{cycle}")
266 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
267 if not self.config.outputStandardsBeforeFinalCycle:
268 self.outputs.remove(f"fgcmStandardStars{cycle}")
271class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
272 pipelineConnections=FgcmFitCycleConnections):
273 """Config for FgcmFitCycle"""
275 doMultipleCycles = pexConfig.Field(
276 doc="Run multiple fit cycles in one task",
277 dtype=bool,
278 default=False,
279 )
280 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
281 doc=("Final cycle number in multiple cycle mode. The initial cycle "
282 "is 0, with limited parameters fit. The next cycle is 1 with "
283 "full parameter fit. The final cycle is a clean-up with no "
284 "parameters fit. There will be a total of "
285 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
286 "cycle number cannot be less than 2."),
287 dtype=int,
288 default=5,
289 min=2,
290 max=MULTIPLE_CYCLES_MAX,
291 inclusiveMax=True,
292 )
293 bands = pexConfig.ListField(
294 doc="Bands to run calibration",
295 dtype=str,
296 default=[],
297 )
298 fitBands = pexConfig.ListField(
299 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
300 "the atmosphere constrained from the 'fitBands' on the same night. "
301 "Must be a subset of `config.bands`"),
302 dtype=str,
303 default=[],
304 )
305 requiredBands = pexConfig.ListField(
306 doc=("Bands that are required for a star to be considered a calibration star. "
307 "Must be a subset of `config.bands`"),
308 dtype=str,
309 default=[],
310 )
311 # The following config will not be necessary after Gen2 retirement.
312 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
313 # is easiest to access in the config file.
314 physicalFilterMap = pexConfig.DictField(
315 doc="Mapping from 'physicalFilter' to band.",
316 keytype=str,
317 itemtype=str,
318 default={},
319 )
320 doReferenceCalibration = pexConfig.Field(
321 doc="Use reference catalog as additional constraint on calibration",
322 dtype=bool,
323 default=True,
324 )
325 refStarSnMin = pexConfig.Field(
326 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
327 dtype=float,
328 default=50.0,
329 )
330 refStarOutlierNSig = pexConfig.Field(
331 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
332 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
333 dtype=float,
334 default=4.0,
335 )
336 applyRefStarColorCuts = pexConfig.Field(
337 doc="Apply color cuts to reference stars?",
338 dtype=bool,
339 default=True,
340 )
341 nCore = pexConfig.Field(
342 doc="Number of cores to use",
343 dtype=int,
344 default=4,
345 )
346 nStarPerRun = pexConfig.Field(
347 doc="Number of stars to run in each chunk",
348 dtype=int,
349 default=200000,
350 )
351 nExpPerRun = pexConfig.Field(
352 doc="Number of exposures to run in each chunk",
353 dtype=int,
354 default=1000,
355 )
356 reserveFraction = pexConfig.Field(
357 doc="Fraction of stars to reserve for testing",
358 dtype=float,
359 default=0.1,
360 )
361 freezeStdAtmosphere = pexConfig.Field(
362 doc="Freeze atmosphere parameters to standard (for testing)",
363 dtype=bool,
364 default=False,
365 )
366 precomputeSuperStarInitialCycle = pexConfig.Field(
367 doc="Precompute superstar flat for initial cycle",
368 dtype=bool,
369 default=False,
370 )
371 superStarSubCcdDict = pexConfig.DictField(
372 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
373 "Must have one entry per band."),
374 keytype=str,
375 itemtype=bool,
376 default={},
377 )
378 superStarSubCcdChebyshevOrder = pexConfig.Field(
379 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
380 "Global default is first-order polynomials, and should be overridden "
381 "on a camera-by-camera basis depending on the ISR."),
382 dtype=int,
383 default=1,
384 )
385 superStarSubCcdTriangular = pexConfig.Field(
386 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
387 "suppress high-order cross terms?"),
388 dtype=bool,
389 default=False,
390 )
391 superStarSigmaClip = pexConfig.Field(
392 doc="Number of sigma to clip outliers when selecting for superstar flats",
393 dtype=float,
394 default=5.0,
395 )
396 focalPlaneSigmaClip = pexConfig.Field(
397 doc="Number of sigma to clip outliers per focal-plane.",
398 dtype=float,
399 default=4.0,
400 )
401 ccdGraySubCcdDict = pexConfig.DictField(
402 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
403 "('ccd gray') on a sub-ccd scale."),
404 keytype=str,
405 itemtype=bool,
406 default={},
407 )
408 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
409 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
410 dtype=int,
411 default=1,
412 )
413 ccdGraySubCcdTriangular = pexConfig.Field(
414 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
415 "suppress high-order cross terms?"),
416 dtype=bool,
417 default=True,
418 )
419 ccdGrayFocalPlaneDict = pexConfig.DictField(
420 doc=("Per-band specification on whether to compute focal-plane residual "
421 "('ccd gray') corrections."),
422 keytype=str,
423 itemtype=bool,
424 default={},
425 )
426 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
427 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
428 "gray corrections. If there are fewer good CCDs then the gray "
429 "correction is computed per-ccd."),
430 dtype=int,
431 default=1,
432 )
433 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
434 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
435 dtype=int,
436 default=3,
437 )
438 cycleNumber = pexConfig.Field(
439 doc=("FGCM fit cycle number. This is automatically incremented after each run "
440 "and stage of outlier rejection. See cookbook for details."),
441 dtype=int,
442 default=None,
443 )
444 isFinalCycle = pexConfig.Field(
445 doc=("Is this the final cycle of the fitting? Will automatically compute final "
446 "selection of stars and photometric exposures, and will output zeropoints "
447 "and standard stars for use in fgcmOutputProducts"),
448 dtype=bool,
449 default=False,
450 )
451 maxIterBeforeFinalCycle = pexConfig.Field(
452 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
453 "will always be 0 in the final cycle for cleanup and final selection."),
454 dtype=int,
455 default=50,
456 )
457 deltaMagBkgOffsetPercentile = pexConfig.Field(
458 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
459 "offset from local background subtraction."),
460 dtype=float,
461 default=0.25,
462 )
463 deltaMagBkgPerCcd = pexConfig.Field(
464 doc=("Compute net offset from local background subtraction per-ccd? "
465 "Otherwise, use computation per visit."),
466 dtype=bool,
467 default=False,
468 )
469 utBoundary = pexConfig.Field(
470 doc="Boundary (in UTC) from day-to-day",
471 dtype=float,
472 default=None,
473 )
474 washMjds = pexConfig.ListField(
475 doc="Mirror wash MJDs",
476 dtype=float,
477 default=(0.0,),
478 )
479 epochMjds = pexConfig.ListField(
480 doc="Epoch boundaries in MJD",
481 dtype=float,
482 default=(0.0,),
483 )
484 minObsPerBand = pexConfig.Field(
485 doc="Minimum good observations per band",
486 dtype=int,
487 default=2,
488 )
489 # TODO: When DM-16511 is done, it will be possible to get the
490 # telescope latitude directly from the camera.
491 latitude = pexConfig.Field(
492 doc="Observatory latitude",
493 dtype=float,
494 default=None,
495 )
496 brightObsGrayMax = pexConfig.Field(
497 doc="Maximum gray extinction to be considered bright observation",
498 dtype=float,
499 default=0.15,
500 )
501 minStarPerCcd = pexConfig.Field(
502 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
503 "CCDs with fewer stars will have their calibration estimated from other "
504 "CCDs in the same visit, with zeropoint error increased accordingly."),
505 dtype=int,
506 default=5,
507 )
508 minCcdPerExp = pexConfig.Field(
509 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
510 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
511 dtype=int,
512 default=5,
513 )
514 maxCcdGrayErr = pexConfig.Field(
515 doc="Maximum error on CCD gray offset to be considered photometric",
516 dtype=float,
517 default=0.05,
518 )
519 minStarPerExp = pexConfig.Field(
520 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
521 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
522 dtype=int,
523 default=600,
524 )
525 minExpPerNight = pexConfig.Field(
526 doc="Minimum number of good exposures/visits to consider a partly photometric night",
527 dtype=int,
528 default=10,
529 )
530 expGrayInitialCut = pexConfig.Field(
531 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
532 "observations."),
533 dtype=float,
534 default=-0.25,
535 )
536 expGrayPhotometricCutDict = pexConfig.DictField(
537 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
538 "('gray term') for a visit to be considered photometric. Must have one "
539 "entry per band. Broad-band filters should be -0.05."),
540 keytype=str,
541 itemtype=float,
542 default={},
543 )
544 expGrayHighCutDict = pexConfig.DictField(
545 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
546 "('gray term') for a visit to be considered photometric. Must have one "
547 "entry per band. Broad-band filters should be 0.2."),
548 keytype=str,
549 itemtype=float,
550 default={},
551 )
552 expGrayRecoverCut = pexConfig.Field(
553 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
554 "Visits with more gray extinction will only get CCD zeropoints if there are "
555 "sufficient star observations (minStarPerCcd) on that CCD."),
556 dtype=float,
557 default=-1.0,
558 )
559 expVarGrayPhotometricCutDict = pexConfig.DictField(
560 doc=("Per-band specification on maximum exposure variance to be considered possibly "
561 "photometric. Must have one entry per band. Broad-band filters should be "
562 "0.0005."),
563 keytype=str,
564 itemtype=float,
565 default={},
566 )
567 expGrayErrRecoverCut = pexConfig.Field(
568 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
569 "Visits with more gray variance will only get CCD zeropoints if there are "
570 "sufficient star observations (minStarPerCcd) on that CCD."),
571 dtype=float,
572 default=0.05,
573 )
574 aperCorrFitNBins = pexConfig.Field(
575 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
576 "no fit will be performed, and the config.aperCorrInputSlopes will be "
577 "used if available."),
578 dtype=int,
579 default=10,
580 )
581 aperCorrInputSlopeDict = pexConfig.DictField(
582 doc=("Per-band specification of aperture correction input slope parameters. These "
583 "are used on the first fit iteration, and aperture correction parameters will "
584 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
585 "to set this when there is insufficient data to fit the parameters (e.g. "
586 "tract mode)."),
587 keytype=str,
588 itemtype=float,
589 default={},
590 )
591 sedboundaryterms = pexConfig.ConfigField(
592 doc="Mapping from bands to SED boundary term names used is sedterms.",
593 dtype=SedboundarytermDict,
594 )
595 sedterms = pexConfig.ConfigField(
596 doc="Mapping from terms to bands for fgcm linear SED approximations.",
597 dtype=SedtermDict,
598 )
599 sigFgcmMaxErr = pexConfig.Field(
600 doc="Maximum mag error for fitting sigma_FGCM",
601 dtype=float,
602 default=0.01,
603 )
604 sigFgcmMaxEGrayDict = pexConfig.DictField(
605 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
606 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
607 "should be 0.05."),
608 keytype=str,
609 itemtype=float,
610 default={},
611 )
612 ccdGrayMaxStarErr = pexConfig.Field(
613 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
614 "computation"),
615 dtype=float,
616 default=0.10,
617 )
618 approxThroughputDict = pexConfig.DictField(
619 doc=("Per-band specification of the approximate overall throughput at the start of "
620 "calibration observations. Must have one entry per band. Typically should "
621 "be 1.0."),
622 keytype=str,
623 itemtype=float,
624 default={},
625 )
626 sigmaCalRange = pexConfig.ListField(
627 doc="Allowed range for systematic error floor estimation",
628 dtype=float,
629 default=(0.001, 0.003),
630 )
631 sigmaCalFitPercentile = pexConfig.ListField(
632 doc="Magnitude percentile range to fit systematic error floor",
633 dtype=float,
634 default=(0.05, 0.15),
635 )
636 sigmaCalPlotPercentile = pexConfig.ListField(
637 doc="Magnitude percentile range to plot systematic error floor",
638 dtype=float,
639 default=(0.05, 0.95),
640 )
641 sigma0Phot = pexConfig.Field(
642 doc="Systematic error floor for all zeropoints",
643 dtype=float,
644 default=0.003,
645 )
646 mapLongitudeRef = pexConfig.Field(
647 doc="Reference longitude for plotting maps",
648 dtype=float,
649 default=0.0,
650 )
651 mapNSide = pexConfig.Field(
652 doc="Healpix nside for plotting maps",
653 dtype=int,
654 default=256,
655 )
656 outfileBase = pexConfig.Field(
657 doc="Filename start for plot output files",
658 dtype=str,
659 default=None,
660 )
661 starColorCuts = pexConfig.ListField(
662 doc="Encoded star-color cuts (to be cleaned up)",
663 dtype=str,
664 default=("NO_DATA",),
665 )
666 colorSplitBands = pexConfig.ListField(
667 doc="Band names to use to split stars by color. Must have 2 entries.",
668 dtype=str,
669 length=2,
670 default=('g', 'i'),
671 )
672 modelMagErrors = pexConfig.Field(
673 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
674 dtype=bool,
675 default=True,
676 )
677 useQuadraticPwv = pexConfig.Field(
678 doc="Model PWV with a quadratic term for variation through the night?",
679 dtype=bool,
680 default=False,
681 )
682 instrumentParsPerBand = pexConfig.Field(
683 doc=("Model instrumental parameters per band? "
684 "Otherwise, instrumental parameters (QE changes with time) are "
685 "shared among all bands."),
686 dtype=bool,
687 default=False,
688 )
689 instrumentSlopeMinDeltaT = pexConfig.Field(
690 doc=("Minimum time change (in days) between observations to use in constraining "
691 "instrument slope."),
692 dtype=float,
693 default=20.0,
694 )
695 fitMirrorChromaticity = pexConfig.Field(
696 doc="Fit (intraband) mirror chromatic term?",
697 dtype=bool,
698 default=False,
699 )
700 coatingMjds = pexConfig.ListField(
701 doc="Mirror coating dates in MJD",
702 dtype=float,
703 default=(0.0,),
704 )
705 outputStandardsBeforeFinalCycle = pexConfig.Field(
706 doc="Output standard stars prior to final cycle? Used in debugging.",
707 dtype=bool,
708 default=False,
709 )
710 outputZeropointsBeforeFinalCycle = pexConfig.Field(
711 doc="Output standard stars prior to final cycle? Used in debugging.",
712 dtype=bool,
713 default=False,
714 )
715 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
716 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
717 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
718 keytype=str,
719 itemtype=bool,
720 default={},
721 )
722 autoPhotometricCutNSig = pexConfig.Field(
723 doc=("Number of sigma for automatic computation of (low) photometric cut. "
724 "Cut is based on exposure gray width (per band), unless "
725 "useRepeatabilityForExpGrayCuts is set, in which case the star "
726 "repeatability is used (also per band)."),
727 dtype=float,
728 default=3.0,
729 )
730 autoHighCutNSig = pexConfig.Field(
731 doc=("Number of sigma for automatic computation of (high) outlier cut. "
732 "Cut is based on exposure gray width (per band), unless "
733 "useRepeatabilityForExpGrayCuts is set, in which case the star "
734 "repeatability is used (also per band)."),
735 dtype=float,
736 default=4.0,
737 )
738 quietMode = pexConfig.Field(
739 doc="Be less verbose with logging.",
740 dtype=bool,
741 default=False,
742 )
743 doPlots = pexConfig.Field(
744 doc="Make fgcm QA plots.",
745 dtype=bool,
746 default=True,
747 )
748 randomSeed = pexConfig.Field(
749 doc="Random seed for fgcm for consistency in tests.",
750 dtype=int,
751 default=None,
752 optional=True,
753 )
755 def validate(self):
756 super().validate()
758 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
759 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
760 raise RuntimeError(msg)
761 if self.connections.cycleNumber != str(self.cycleNumber):
762 msg = "cycleNumber in template must be equal to connections.cycleNumber"
763 raise RuntimeError(msg)
765 for band in self.fitBands:
766 if band not in self.bands:
767 msg = 'fitBand %s not in bands' % (band)
768 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
769 for band in self.requiredBands:
770 if band not in self.bands:
771 msg = 'requiredBand %s not in bands' % (band)
772 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
773 for band in self.colorSplitBands:
774 if band not in self.bands:
775 msg = 'colorSplitBand %s not in bands' % (band)
776 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
777 for band in self.bands:
778 if band not in self.superStarSubCcdDict:
779 msg = 'band %s not in superStarSubCcdDict' % (band)
780 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
781 self, msg)
782 if band not in self.ccdGraySubCcdDict:
783 msg = 'band %s not in ccdGraySubCcdDict' % (band)
784 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
785 self, msg)
786 if band not in self.expGrayPhotometricCutDict:
787 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
788 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
789 self, msg)
790 if band not in self.expGrayHighCutDict:
791 msg = 'band %s not in expGrayHighCutDict' % (band)
792 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
793 self, msg)
794 if band not in self.expVarGrayPhotometricCutDict:
795 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
796 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
797 self, msg)
798 if band not in self.sigFgcmMaxEGrayDict:
799 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
800 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
801 self, msg)
802 if band not in self.approxThroughputDict:
803 msg = 'band %s not in approxThroughputDict' % (band)
804 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
805 self, msg)
806 if band not in self.useRepeatabilityForExpGrayCutsDict:
807 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
808 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
809 self, msg)
812class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
813 """Subclass of TaskRunner for fgcmFitCycleTask
815 fgcmFitCycleTask.run() takes one argument, the butler, and uses
816 stars and visits previously extracted from dataRefs by
817 fgcmBuildStars.
818 This Runner does not perform any dataRef parallelization, but the FGCM
819 code called by the Task uses python multiprocessing (see the "ncores"
820 config option).
821 """
823 @staticmethod
824 def getTargetList(parsedCmd):
825 """
826 Return a list with one element, the butler.
827 """
828 return [parsedCmd.butler]
830 def __call__(self, butler):
831 """
832 Parameters
833 ----------
834 butler: `lsst.daf.persistence.Butler`
836 Returns
837 -------
838 exitStatus: `list` with `pipeBase.Struct`
839 exitStatus (0: success; 1: failure)
840 """
842 task = self.TaskClass(config=self.config, log=self.log)
844 exitStatus = 0
845 if self.doRaise:
846 task.runDataRef(butler)
847 else:
848 try:
849 task.runDataRef(butler)
850 except Exception as e:
851 exitStatus = 1
852 task.log.fatal("Failed: %s" % e)
853 if not isinstance(e, pipeBase.TaskError):
854 traceback.print_exc(file=sys.stderr)
856 task.writeMetadata(butler)
858 # The task does not return any results:
859 return [pipeBase.Struct(exitStatus=exitStatus)]
861 def run(self, parsedCmd):
862 """
863 Run the task, with no multiprocessing
865 Parameters
866 ----------
867 parsedCmd: ArgumentParser parsed command line
868 """
870 resultList = []
872 if self.precall(parsedCmd):
873 targetList = self.getTargetList(parsedCmd)
874 # make sure that we only get 1
875 resultList = self(targetList[0])
877 return resultList
880class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
881 """
882 Run Single fit cycle for FGCM global calibration
883 """
885 ConfigClass = FgcmFitCycleConfig
886 RunnerClass = FgcmFitCycleRunner
887 _DefaultName = "fgcmFitCycle"
889 def __init__(self, butler=None, initInputs=None, **kwargs):
890 super().__init__(**kwargs)
892 # no saving of metadata for now
893 def _getMetadataName(self):
894 return None
896 def runQuantum(self, butlerQC, inputRefs, outputRefs):
897 camera = butlerQC.get(inputRefs.camera)
899 dataRefDict = {}
901 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
902 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
903 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
904 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
905 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
906 if self.config.doReferenceCalibration:
907 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
908 if self.config.cycleNumber > 0:
909 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
910 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
912 fgcmDatasetDict = None
913 if self.config.doMultipleCycles:
914 # Run multiple cycles at once.
915 config = copy.copy(self.config)
916 config.update(cycleNumber=0)
917 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
918 if cycle == self.config.multipleCyclesFinalCycleNumber:
919 config.update(isFinalCycle=True)
921 if cycle > 0:
922 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
923 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
925 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
926 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
927 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
928 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
929 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
930 if self.outputZeropoints:
931 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
932 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
933 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
934 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
935 if self.outputStandards:
936 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
937 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
938 else:
939 # Run a single cycle
940 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
942 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
943 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
944 if self.outputZeropoints:
945 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
946 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
947 if self.outputStandards:
948 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
950 @timeMethod
951 def runDataRef(self, butler):
952 """
953 Run a single fit cycle for FGCM
955 Parameters
956 ----------
957 butler: `lsst.daf.persistence.Butler`
958 """
959 self._checkDatasetsExist(butler)
961 dataRefDict = {}
962 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
963 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
964 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
965 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
966 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
967 if self.config.doReferenceCalibration:
968 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
969 if self.config.cycleNumber > 0:
970 lastCycle = self.config.cycleNumber - 1
971 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
972 fgcmcycle=lastCycle)
973 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
974 fgcmcycle=lastCycle)
976 camera = butler.get('camera')
977 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
979 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
980 fgcmcycle=self.config.cycleNumber)
981 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
982 fgcmcycle=self.config.cycleNumber)
983 if self.outputZeropoints:
984 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
985 fgcmcycle=self.config.cycleNumber)
986 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
987 fgcmcycle=self.config.cycleNumber)
988 if self.outputStandards:
989 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
990 fgcmcycle=self.config.cycleNumber)
992 def writeConfig(self, butler, clobber=False, doBackup=True):
993 """Write the configuration used for processing the data, or check that an existing
994 one is equal to the new one if present. This is an override of the regular
995 version from pipe_base that knows about fgcmcycle.
997 Parameters
998 ----------
999 butler : `lsst.daf.persistence.Butler`
1000 Data butler used to write the config. The config is written to dataset type
1001 `CmdLineTask._getConfigName`.
1002 clobber : `bool`, optional
1003 A boolean flag that controls what happens if a config already has been saved:
1004 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1005 - `False`: raise `TaskError` if this config does not match the existing config.
1006 doBackup : `bool`, optional
1007 Set to `True` to backup the config files if clobbering.
1008 """
1009 configName = self._getConfigName()
1010 if configName is None:
1011 return
1012 if clobber:
1013 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1014 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1015 # this may be subject to a race condition; see #2789
1016 try:
1017 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1018 except Exception as exc:
1019 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1020 (configName, exc))
1022 def logConfigMismatch(msg):
1023 self.log.fatal("Comparing configuration: %s", msg)
1025 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1026 raise pipeBase.TaskError(
1027 f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1028 " must be consistent within the same output repo (override with --clobber-config)")
1029 else:
1030 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1032 def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1033 """
1034 Run the fit cycle
1036 Parameters
1037 ----------
1038 camera : `lsst.afw.cameraGeom.Camera`
1039 dataRefDict : `dict`
1040 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1041 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1042 dataRef dictionary with keys:
1044 ``"fgcmLookUpTable"``
1045 dataRef for the FGCM look-up table.
1046 ``"fgcmVisitCatalog"``
1047 dataRef for visit summary catalog.
1048 ``"fgcmStarObservations"``
1049 dataRef for star observation catalog.
1050 ``"fgcmStarIds"``
1051 dataRef for star id catalog.
1052 ``"fgcmStarIndices"``
1053 dataRef for star index catalog.
1054 ``"fgcmReferenceStars"``
1055 dataRef for matched reference star catalog.
1056 ``"fgcmFlaggedStars"``
1057 dataRef for flagged star catalog.
1058 ``"fgcmFitParameters"``
1059 dataRef for fit parameter catalog.
1060 config : `lsst.pex.config.Config`, optional
1061 Configuration to use to override self.config.
1063 Returns
1064 -------
1065 fgcmDatasetDict : `dict`
1066 Dictionary of datasets to persist.
1067 """
1068 if config is not None:
1069 _config = config
1070 else:
1071 _config = self.config
1073 # Set defaults on whether to output standards and zeropoints
1074 self.maxIter = _config.maxIterBeforeFinalCycle
1075 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1076 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1077 self.resetFitParameters = True
1079 if _config.isFinalCycle:
1080 # This is the final fit cycle, so we do not want to reset fit
1081 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1082 # and we always want to output standards and zeropoints
1083 self.maxIter = 0
1084 self.outputStandards = True
1085 self.outputZeropoints = True
1086 self.resetFitParameters = False
1088 lutCat = dataRefDict['fgcmLookUpTable'].get()
1089 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1090 dict(_config.physicalFilterMap))
1091 del lutCat
1093 configDict = makeConfigDict(_config, self.log, camera,
1094 self.maxIter, self.resetFitParameters,
1095 self.outputZeropoints,
1096 lutIndexVals[0]['FILTERNAMES'])
1098 # next we need the exposure/visit information
1099 visitCat = dataRefDict['fgcmVisitCatalog'].get()
1100 fgcmExpInfo = translateVisitCatalog(visitCat)
1101 del visitCat
1103 # Use the first orientation.
1104 # TODO: DM-21215 will generalize to arbitrary camera orientations
1105 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1107 noFitsDict = {'lutIndex': lutIndexVals,
1108 'lutStd': lutStd,
1109 'expInfo': fgcmExpInfo,
1110 'ccdOffsets': ccdOffsets}
1112 # set up the fitter object
1113 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1114 noFitsDict=noFitsDict, noOutput=True)
1116 # create the parameter object
1117 if (fgcmFitCycle.initialCycle):
1118 # cycle = 0, initial cycle
1119 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1120 fgcmLut,
1121 fgcmExpInfo)
1122 else:
1123 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog):
1124 parCat = dataRefDict['fgcmFitParameters']
1125 else:
1126 parCat = dataRefDict['fgcmFitParameters'].get()
1127 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1128 del parCat
1129 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1130 fgcmExpInfo,
1131 inParInfo,
1132 inParams,
1133 inSuperStar)
1135 # set up the stars...
1136 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1138 starObs = dataRefDict['fgcmStarObservations'].get()
1139 starIds = dataRefDict['fgcmStarIds'].get()
1140 starIndices = dataRefDict['fgcmStarIndices'].get()
1142 # grab the flagged stars if available
1143 if 'fgcmFlaggedStars' in dataRefDict:
1144 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1145 flaggedStars = dataRefDict['fgcmFlaggedStars']
1146 else:
1147 flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1148 flagId = flaggedStars['objId'][:]
1149 flagFlag = flaggedStars['objFlag'][:]
1150 else:
1151 flaggedStars = None
1152 flagId = None
1153 flagFlag = None
1155 if _config.doReferenceCalibration:
1156 refStars = dataRefDict['fgcmReferenceStars'].get()
1158 refMag, refMagErr = extractReferenceMags(refStars,
1159 _config.bands,
1160 _config.physicalFilterMap)
1161 refId = refStars['fgcm_id'][:]
1162 else:
1163 refStars = None
1164 refId = None
1165 refMag = None
1166 refMagErr = None
1168 # match star observations to visits
1169 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1170 # actually be transferred into fgcm using the indexing below.
1171 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1173 # The fgcmStars.loadStars method will copy all the star information into
1174 # special shared memory objects that will not blow up the memory usage when
1175 # used with python multiprocessing. Once all the numbers are copied,
1176 # it is necessary to release all references to the objects that previously
1177 # stored the data to ensure that the garbage collector can clear the memory,
1178 # and ensure that this memory is not copied when multiprocessing kicks in.
1180 # We determine the conversion from the native units (typically radians) to
1181 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1182 # numpy arrays rather than Angles, which would we approximately 600x slower.
1183 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1185 fgcmStars.loadStars(fgcmPars,
1186 starObs['visit'][starIndices['obsIndex']],
1187 starObs['ccd'][starIndices['obsIndex']],
1188 starObs['ra'][starIndices['obsIndex']] * conv,
1189 starObs['dec'][starIndices['obsIndex']] * conv,
1190 starObs['instMag'][starIndices['obsIndex']],
1191 starObs['instMagErr'][starIndices['obsIndex']],
1192 fgcmExpInfo['FILTERNAME'][visitIndex],
1193 starIds['fgcm_id'][:],
1194 starIds['ra'][:],
1195 starIds['dec'][:],
1196 starIds['obsArrIndex'][:],
1197 starIds['nObs'][:],
1198 obsX=starObs['x'][starIndices['obsIndex']],
1199 obsY=starObs['y'][starIndices['obsIndex']],
1200 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1201 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1202 refID=refId,
1203 refMag=refMag,
1204 refMagErr=refMagErr,
1205 flagID=flagId,
1206 flagFlag=flagFlag,
1207 computeNobs=True)
1209 # Release all references to temporary objects holding star data (see above)
1210 del starObs
1211 del starIds
1212 del starIndices
1213 del flagId
1214 del flagFlag
1215 del flaggedStars
1216 del refStars
1217 del refId
1218 del refMag
1219 del refMagErr
1221 # and set the bits in the cycle object
1222 fgcmFitCycle.setLUT(fgcmLut)
1223 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1224 fgcmFitCycle.setPars(fgcmPars)
1226 # finish the setup
1227 fgcmFitCycle.finishSetup()
1229 # and run
1230 fgcmFitCycle.run()
1232 ##################
1233 # Persistance
1234 ##################
1236 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1238 # Output the config for the next cycle
1239 # We need to make a copy since the input one has been frozen
1241 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1242 i, b in enumerate(_config.bands)}
1243 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1244 i, band in enumerate(_config.bands)}
1246 outConfig = copy.copy(_config)
1247 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1248 precomputeSuperStarInitialCycle=False,
1249 freezeStdAtmosphere=False,
1250 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1251 expGrayHighCutDict=updatedHighCutDict)
1253 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1254 cycleNumber=str(_config.cycleNumber + 1))
1256 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1257 outConfig.cycleNumber)
1258 outConfig.save(configFileName)
1260 if _config.isFinalCycle == 1:
1261 # We are done, ready to output products
1262 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1263 else:
1264 self.log.info("Saved config for next cycle to %s" % (configFileName))
1265 self.log.info("Be sure to look at:")
1266 self.log.info(" config.expGrayPhotometricCut")
1267 self.log.info(" config.expGrayHighCut")
1268 self.log.info("If you are satisfied with the fit, please set:")
1269 self.log.info(" config.isFinalCycle = True")
1271 fgcmFitCycle.freeSharedMemory()
1273 return fgcmDatasetDict, outConfig
1275 def _checkDatasetsExist(self, butler):
1276 """
1277 Check if necessary datasets exist to run fgcmFitCycle
1279 Parameters
1280 ----------
1281 butler: `lsst.daf.persistence.Butler`
1283 Raises
1284 ------
1285 RuntimeError
1286 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1287 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1288 If cycleNumber > 0, then also checks for fgcmFitParameters,
1289 fgcmFlaggedStars.
1290 """
1292 if not butler.datasetExists('fgcmVisitCatalog'):
1293 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1294 if not butler.datasetExists('fgcmStarObservations'):
1295 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1296 if not butler.datasetExists('fgcmStarIds'):
1297 raise RuntimeError("Could not find fgcmStarIds in repo!")
1298 if not butler.datasetExists('fgcmStarIndices'):
1299 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1300 if not butler.datasetExists('fgcmLookUpTable'):
1301 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1303 # Need additional datasets if we are not the initial cycle
1304 if (self.config.cycleNumber > 0):
1305 if not butler.datasetExists('fgcmFitParameters',
1306 fgcmcycle=self.config.cycleNumber-1):
1307 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1308 (self.config.cycleNumber-1))
1309 if not butler.datasetExists('fgcmFlaggedStars',
1310 fgcmcycle=self.config.cycleNumber-1):
1311 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1312 (self.config.cycleNumber-1))
1314 # And additional dataset if we want reference calibration
1315 if self.config.doReferenceCalibration:
1316 if not butler.datasetExists('fgcmReferenceStars'):
1317 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1318 "doReferenceCalibration is True.")
1320 def _loadParameters(self, parCat):
1321 """
1322 Load FGCM parameters from a previous fit cycle
1324 Parameters
1325 ----------
1326 parCat : `lsst.afw.table.BaseCatalog`
1327 Parameter catalog in afw table form.
1329 Returns
1330 -------
1331 inParInfo: `numpy.ndarray`
1332 Numpy array parameter information formatted for input to fgcm
1333 inParameters: `numpy.ndarray`
1334 Numpy array parameter values formatted for input to fgcm
1335 inSuperStar: `numpy.array`
1336 Superstar flat formatted for input to fgcm
1337 """
1338 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1339 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1341 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1342 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1343 (parLutFilterNames.size, )),
1344 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1345 ('LNTAUUNIT', 'f8'),
1346 ('LNTAUSLOPEUNIT', 'f8'),
1347 ('ALPHAUNIT', 'f8'),
1348 ('LNPWVUNIT', 'f8'),
1349 ('LNPWVSLOPEUNIT', 'f8'),
1350 ('LNPWVQUADRATICUNIT', 'f8'),
1351 ('LNPWVGLOBALUNIT', 'f8'),
1352 ('O3UNIT', 'f8'),
1353 ('QESYSUNIT', 'f8'),
1354 ('FILTEROFFSETUNIT', 'f8'),
1355 ('HASEXTERNALPWV', 'i2'),
1356 ('HASEXTERNALTAU', 'i2')])
1357 inParInfo['NCCD'] = parCat['nCcd']
1358 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1359 inParInfo['FITBANDS'][:] = parFitBands
1360 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1361 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1363 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1364 ('PARO3', 'f8', (parCat['parO3'].size, )),
1365 ('PARLNTAUINTERCEPT', 'f8',
1366 (parCat['parLnTauIntercept'].size, )),
1367 ('PARLNTAUSLOPE', 'f8',
1368 (parCat['parLnTauSlope'].size, )),
1369 ('PARLNPWVINTERCEPT', 'f8',
1370 (parCat['parLnPwvIntercept'].size, )),
1371 ('PARLNPWVSLOPE', 'f8',
1372 (parCat['parLnPwvSlope'].size, )),
1373 ('PARLNPWVQUADRATIC', 'f8',
1374 (parCat['parLnPwvQuadratic'].size, )),
1375 ('PARQESYSINTERCEPT', 'f8',
1376 (parCat['parQeSysIntercept'].size, )),
1377 ('COMPQESYSSLOPE', 'f8',
1378 (parCat['compQeSysSlope'].size, )),
1379 ('PARFILTEROFFSET', 'f8',
1380 (parCat['parFilterOffset'].size, )),
1381 ('PARFILTEROFFSETFITFLAG', 'i2',
1382 (parCat['parFilterOffsetFitFlag'].size, )),
1383 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1384 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1385 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1386 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1387 ('COMPABSTHROUGHPUT', 'f8',
1388 (parCat['compAbsThroughput'].size, )),
1389 ('COMPREFOFFSET', 'f8',
1390 (parCat['compRefOffset'].size, )),
1391 ('COMPREFSIGMA', 'f8',
1392 (parCat['compRefSigma'].size, )),
1393 ('COMPMIRRORCHROMATICITY', 'f8',
1394 (parCat['compMirrorChromaticity'].size, )),
1395 ('MIRRORCHROMATICITYPIVOT', 'f8',
1396 (parCat['mirrorChromaticityPivot'].size, )),
1397 ('COMPMEDIANSEDSLOPE', 'f8',
1398 (parCat['compMedianSedSlope'].size, )),
1399 ('COMPAPERCORRPIVOT', 'f8',
1400 (parCat['compAperCorrPivot'].size, )),
1401 ('COMPAPERCORRSLOPE', 'f8',
1402 (parCat['compAperCorrSlope'].size, )),
1403 ('COMPAPERCORRSLOPEERR', 'f8',
1404 (parCat['compAperCorrSlopeErr'].size, )),
1405 ('COMPAPERCORRRANGE', 'f8',
1406 (parCat['compAperCorrRange'].size, )),
1407 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1408 (parCat['compModelErrExptimePivot'].size, )),
1409 ('COMPMODELERRFWHMPIVOT', 'f8',
1410 (parCat['compModelErrFwhmPivot'].size, )),
1411 ('COMPMODELERRSKYPIVOT', 'f8',
1412 (parCat['compModelErrSkyPivot'].size, )),
1413 ('COMPMODELERRPARS', 'f8',
1414 (parCat['compModelErrPars'].size, )),
1415 ('COMPEXPGRAY', 'f8',
1416 (parCat['compExpGray'].size, )),
1417 ('COMPVARGRAY', 'f8',
1418 (parCat['compVarGray'].size, )),
1419 ('COMPEXPDELTAMAGBKG', 'f8',
1420 (parCat['compExpDeltaMagBkg'].size, )),
1421 ('COMPNGOODSTARPEREXP', 'i4',
1422 (parCat['compNGoodStarPerExp'].size, )),
1423 ('COMPSIGFGCM', 'f8',
1424 (parCat['compSigFgcm'].size, )),
1425 ('COMPSIGMACAL', 'f8',
1426 (parCat['compSigmaCal'].size, )),
1427 ('COMPRETRIEVEDLNPWV', 'f8',
1428 (parCat['compRetrievedLnPwv'].size, )),
1429 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1430 (parCat['compRetrievedLnPwvRaw'].size, )),
1431 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1432 (parCat['compRetrievedLnPwvFlag'].size, )),
1433 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1434 (parCat['compRetrievedTauNight'].size, ))])
1436 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1437 inParams['PARO3'][:] = parCat['parO3'][0, :]
1438 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1439 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1440 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1441 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1442 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1443 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1444 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1445 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1446 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1447 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1448 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1449 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1450 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1451 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1452 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1453 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1454 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1455 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1456 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1457 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1458 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1459 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1460 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1461 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1462 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1463 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1464 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1465 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1466 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1467 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1468 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1469 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1470 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1471 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1472 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1473 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1475 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1476 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1478 return (inParInfo, inParams, inSuperStar)
1480 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1481 """
1482 Persist FGCM datasets through the butler.
1484 Parameters
1485 ----------
1486 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1487 Fgcm Fit cycle object
1488 """
1489 fgcmDatasetDict = {}
1491 # Save the parameters
1492 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1494 parSchema = afwTable.Schema()
1496 comma = ','
1497 lutFilterNameString = comma.join([n.decode('utf-8')
1498 for n in parInfo['LUTFILTERNAMES'][0]])
1499 fitBandString = comma.join([n.decode('utf-8')
1500 for n in parInfo['FITBANDS'][0]])
1502 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1503 lutFilterNameString, fitBandString)
1504 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1505 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1506 lutFilterNameString, fitBandString)
1508 fgcmDatasetDict['fgcmFitParameters'] = parCat
1510 # Save the indices of the flagged stars
1511 # (stars that have been (a) reserved from the fit for testing and
1512 # (b) bad stars that have failed quality checks.)
1513 flagStarSchema = self._makeFlagStarSchema()
1514 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1515 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1517 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1519 # Save the zeropoint information and atmospheres only if desired
1520 if self.outputZeropoints:
1521 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1522 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1524 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1525 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1527 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1529 # Save atmosphere values
1530 # These are generated by the same code that generates zeropoints
1531 atmSchema = makeAtmSchema()
1532 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1534 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1536 # Save the standard stars (if configured)
1537 if self.outputStandards:
1538 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1539 stdSchema = makeStdSchema(len(goodBands))
1540 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1542 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1544 return fgcmDatasetDict
1546 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1547 lutFilterNameString, fitBandString):
1548 """
1549 Make the parameter persistence schema
1551 Parameters
1552 ----------
1553 parInfo: `numpy.ndarray`
1554 Parameter information returned by fgcm
1555 pars: `numpy.ndarray`
1556 Parameter values returned by fgcm
1557 parSuperStarFlat: `numpy.array`
1558 Superstar flat values returned by fgcm
1559 lutFilterNameString: `str`
1560 Combined string of all the lutFilterNames
1561 fitBandString: `str`
1562 Combined string of all the fitBands
1564 Returns
1565 -------
1566 parSchema: `afwTable.schema`
1567 """
1569 parSchema = afwTable.Schema()
1571 # parameter info section
1572 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1573 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1574 size=len(lutFilterNameString))
1575 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1576 size=len(fitBandString))
1577 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1578 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1579 doc='Step units for ln(AOD) slope')
1580 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1581 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1582 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1583 doc='Step units for ln(pwv) slope')
1584 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1585 doc='Step units for ln(pwv) quadratic term')
1586 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1587 doc='Step units for global ln(pwv) parameters')
1588 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1589 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1590 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1591 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1592 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1594 # parameter section
1595 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1596 size=pars['PARALPHA'].size)
1597 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1598 size=pars['PARO3'].size)
1599 parSchema.addField('parLnTauIntercept', type='ArrayD',
1600 doc='ln(Tau) intercept parameter vector',
1601 size=pars['PARLNTAUINTERCEPT'].size)
1602 parSchema.addField('parLnTauSlope', type='ArrayD',
1603 doc='ln(Tau) slope parameter vector',
1604 size=pars['PARLNTAUSLOPE'].size)
1605 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1606 size=pars['PARLNPWVINTERCEPT'].size)
1607 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1608 size=pars['PARLNPWVSLOPE'].size)
1609 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1610 size=pars['PARLNPWVQUADRATIC'].size)
1611 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1612 size=pars['PARQESYSINTERCEPT'].size)
1613 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1614 size=pars[0]['COMPQESYSSLOPE'].size)
1615 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1616 size=pars['PARFILTEROFFSET'].size)
1617 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1618 size=pars['PARFILTEROFFSETFITFLAG'].size)
1619 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1620 doc='Global scale for retrieved ln(pwv)')
1621 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1622 doc='Global offset for retrieved ln(pwv)')
1623 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1624 doc='Nightly offset for retrieved ln(pwv)',
1625 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1626 parSchema.addField('compAbsThroughput', type='ArrayD',
1627 doc='Absolute throughput (relative to transmission curves)',
1628 size=pars['COMPABSTHROUGHPUT'].size)
1629 parSchema.addField('compRefOffset', type='ArrayD',
1630 doc='Offset between reference stars and calibrated stars',
1631 size=pars['COMPREFOFFSET'].size)
1632 parSchema.addField('compRefSigma', type='ArrayD',
1633 doc='Width of reference star/calibrated star distribution',
1634 size=pars['COMPREFSIGMA'].size)
1635 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1636 doc='Computed mirror chromaticity terms',
1637 size=pars['COMPMIRRORCHROMATICITY'].size)
1638 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1639 doc='Mirror chromaticity pivot mjd',
1640 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1641 parSchema.addField('compMedianSedSlope', type='ArrayD',
1642 doc='Computed median SED slope (per band)',
1643 size=pars['COMPMEDIANSEDSLOPE'].size)
1644 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1645 size=pars['COMPAPERCORRPIVOT'].size)
1646 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1647 size=pars['COMPAPERCORRSLOPE'].size)
1648 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1649 size=pars['COMPAPERCORRSLOPEERR'].size)
1650 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1651 size=pars['COMPAPERCORRRANGE'].size)
1652 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1653 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1654 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1655 size=pars['COMPMODELERRFWHMPIVOT'].size)
1656 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1657 size=pars['COMPMODELERRSKYPIVOT'].size)
1658 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1659 size=pars['COMPMODELERRPARS'].size)
1660 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1661 size=pars['COMPEXPGRAY'].size)
1662 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1663 size=pars['COMPVARGRAY'].size)
1664 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1665 doc='Computed exposure offset due to background',
1666 size=pars['COMPEXPDELTAMAGBKG'].size)
1667 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1668 doc='Computed number of good stars per exposure',
1669 size=pars['COMPNGOODSTARPEREXP'].size)
1670 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1671 size=pars['COMPSIGFGCM'].size)
1672 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1673 size=pars['COMPSIGMACAL'].size)
1674 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1675 size=pars['COMPRETRIEVEDLNPWV'].size)
1676 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1677 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1678 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1679 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1680 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1681 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1682 # superstarflat section
1683 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1684 size=4)
1685 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1686 size=parSuperStarFlat.size)
1688 return parSchema
1690 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1691 lutFilterNameString, fitBandString):
1692 """
1693 Make the FGCM parameter catalog for persistence
1695 Parameters
1696 ----------
1697 parSchema: `lsst.afw.table.Schema`
1698 Parameter catalog schema
1699 pars: `numpy.ndarray`
1700 FGCM parameters to put into parCat
1701 parSuperStarFlat: `numpy.array`
1702 FGCM superstar flat array to put into parCat
1703 lutFilterNameString: `str`
1704 Combined string of all the lutFilterNames
1705 fitBandString: `str`
1706 Combined string of all the fitBands
1708 Returns
1709 -------
1710 parCat: `afwTable.BasicCatalog`
1711 Atmosphere and instrumental model parameter catalog for persistence
1712 """
1714 parCat = afwTable.BaseCatalog(parSchema)
1715 parCat.reserve(1)
1717 # The parameter catalog just has one row, with many columns for all the
1718 # atmosphere and instrument fit parameters
1719 rec = parCat.addNew()
1721 # info section
1722 rec['nCcd'] = parInfo['NCCD']
1723 rec['lutFilterNames'] = lutFilterNameString
1724 rec['fitBands'] = fitBandString
1725 # note these are not currently supported here.
1726 rec['hasExternalPwv'] = 0
1727 rec['hasExternalTau'] = 0
1729 # parameter section
1731 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1733 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1734 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1735 'parQeSysIntercept', 'compQeSysSlope',
1736 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1737 'parFilterOffset', 'parFilterOffsetFitFlag',
1738 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1739 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1740 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1741 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1742 'compModelErrSkyPivot', 'compModelErrPars',
1743 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1744 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1745 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1746 'compRetrievedTauNight']
1748 for scalarName in scalarNames:
1749 rec[scalarName] = pars[scalarName.upper()]
1751 for arrName in arrNames:
1752 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1754 # superstar section
1755 rec['superstarSize'][:] = parSuperStarFlat.shape
1756 rec['superstar'][:] = parSuperStarFlat.flatten()
1758 return parCat
1760 def _makeFlagStarSchema(self):
1761 """
1762 Make the flagged-stars schema
1764 Returns
1765 -------
1766 flagStarSchema: `lsst.afw.table.Schema`
1767 """
1769 flagStarSchema = afwTable.Schema()
1771 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1772 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1774 return flagStarSchema
1776 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1777 """
1778 Make the flagged star catalog for persistence
1780 Parameters
1781 ----------
1782 flagStarSchema: `lsst.afw.table.Schema`
1783 Flagged star schema
1784 flagStarStruct: `numpy.ndarray`
1785 Flagged star structure from fgcm
1787 Returns
1788 -------
1789 flagStarCat: `lsst.afw.table.BaseCatalog`
1790 Flagged star catalog for persistence
1791 """
1793 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1794 flagStarCat.resize(flagStarStruct.size)
1796 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1797 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1799 return flagStarCat