lsst.fgcmcal g0047a03da4+e815273cf9
fgcmFitCycle.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
24
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
29
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
35
36import copy
37
38import numpy as np
39
40import lsst.pex.config as pexConfig
41import lsst.pipe.base as pipeBase
42from lsst.pipe.base import connectionTypes
43import lsst.afw.table as afwTable
44
45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
46from .utilities import extractReferenceMags
47from .utilities import makeZptSchema, makeZptCat
48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
49from .sedterms import SedboundarytermDict, SedtermDict
50from .utilities import lookupStaticCalibrations
51from .focalPlaneProjector import FocalPlaneProjector
52
53import fgcm
54
55__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask']
56
57MULTIPLE_CYCLES_MAX = 10
58
59
60class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
61 dimensions=("instrument",),
62 defaultTemplates={"previousCycleNumber": "-1",
63 "cycleNumber": "0"}):
64 camera = connectionTypes.PrerequisiteInput(
65 doc="Camera instrument",
66 name="camera",
67 storageClass="Camera",
68 dimensions=("instrument",),
69 lookupFunction=lookupStaticCalibrations,
70 isCalibration=True,
71 )
72
73 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
74 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
75 "chromatic corrections."),
76 name="fgcmLookUpTable",
77 storageClass="Catalog",
78 dimensions=("instrument",),
79 deferLoad=True,
80 )
81
82 fgcmVisitCatalog = connectionTypes.Input(
83 doc="Catalog of visit information for fgcm",
84 name="fgcmVisitCatalog",
85 storageClass="Catalog",
86 dimensions=("instrument",),
87 deferLoad=True,
88 )
89
90 fgcmStarObservations = connectionTypes.Input(
91 doc="Catalog of star observations for fgcm",
92 name="fgcmStarObservations",
93 storageClass="Catalog",
94 dimensions=("instrument",),
95 deferLoad=True,
96 )
97
98 fgcmStarIds = connectionTypes.Input(
99 doc="Catalog of fgcm calibration star IDs",
100 name="fgcmStarIds",
101 storageClass="Catalog",
102 dimensions=("instrument",),
103 deferLoad=True,
104 )
105
106 fgcmStarIndices = connectionTypes.Input(
107 doc="Catalog of fgcm calibration star indices",
108 name="fgcmStarIndices",
109 storageClass="Catalog",
110 dimensions=("instrument",),
111 deferLoad=True,
112 )
113
114 fgcmReferenceStars = connectionTypes.Input(
115 doc="Catalog of fgcm-matched reference stars",
116 name="fgcmReferenceStars",
117 storageClass="Catalog",
118 dimensions=("instrument",),
119 deferLoad=True,
120 )
121
122 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
123 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
124 name="fgcmFlaggedStars{previousCycleNumber}",
125 storageClass="Catalog",
126 dimensions=("instrument",),
127 deferLoad=True,
128 )
129
130 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
131 doc="Catalog of fgcm fit parameters from previous fit cycle",
132 name="fgcmFitParameters{previousCycleNumber}",
133 storageClass="Catalog",
134 dimensions=("instrument",),
135 deferLoad=True,
136 )
137
138 fgcmFitParameters = connectionTypes.Output(
139 doc="Catalog of fgcm fit parameters from current fit cycle",
140 name="fgcmFitParameters{cycleNumber}",
141 storageClass="Catalog",
142 dimensions=("instrument",),
143 )
144
145 fgcmFlaggedStars = connectionTypes.Output(
146 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
147 name="fgcmFlaggedStars{cycleNumber}",
148 storageClass="Catalog",
149 dimensions=("instrument",),
150 )
151
152 fgcmZeropoints = connectionTypes.Output(
153 doc="Catalog of fgcm zeropoint data from current fit cycle",
154 name="fgcmZeropoints{cycleNumber}",
155 storageClass="Catalog",
156 dimensions=("instrument",),
157 )
158
159 fgcmAtmosphereParameters = connectionTypes.Output(
160 doc="Catalog of atmospheric fit parameters from current fit cycle",
161 name="fgcmAtmosphereParameters{cycleNumber}",
162 storageClass="Catalog",
163 dimensions=("instrument",),
164 )
165
166 fgcmStandardStars = connectionTypes.Output(
167 doc="Catalog of standard star magnitudes from current fit cycle",
168 name="fgcmStandardStars{cycleNumber}",
169 storageClass="SimpleCatalog",
170 dimensions=("instrument",),
171 )
172
173 # Add connections for running multiple cycles
174 # This uses vars() to alter the class __dict__ to programmatically
175 # write many similar outputs.
176 for cycle in range(MULTIPLE_CYCLES_MAX):
177 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
178 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
179 name=f"fgcmFitParameters{cycle}",
180 storageClass="Catalog",
181 dimensions=("instrument",),
182 )
183 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
184 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
185 name=f"fgcmFlaggedStars{cycle}",
186 storageClass="Catalog",
187 dimensions=("instrument",),
188 )
189 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
190 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
191 name=f"fgcmZeropoints{cycle}",
192 storageClass="Catalog",
193 dimensions=("instrument",),
194 )
195 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
196 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
197 name=f"fgcmAtmosphereParameters{cycle}",
198 storageClass="Catalog",
199 dimensions=("instrument",),
200 )
201 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
202 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
203 name=f"fgcmStandardStars{cycle}",
204 storageClass="SimpleCatalog",
205 dimensions=("instrument",),
206 )
207
208 def __init__(self, *, config=None):
209 super().__init__(config=config)
210
211 if not config.doReferenceCalibration:
212 self.inputs.remove("fgcmReferenceStars")
213
214 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
215 raise ValueError("cycleNumber must be of integer format")
216 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
217 raise ValueError("previousCycleNumber must be of integer format")
218 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
219 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
220
221 if int(config.connections.cycleNumber) == 0:
222 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
223 self.prerequisiteInputs.remove("fgcmFitParametersInput")
224
225 if not self.config.doMultipleCycles:
226 # Single-cycle run
227 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
228 self.outputs.remove("fgcmStandardStars")
229
230 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
231 self.outputs.remove("fgcmZeropoints")
232 self.outputs.remove("fgcmAtmosphereParameters")
233
234 # Remove all multiple cycle outputs
235 for cycle in range(0, MULTIPLE_CYCLES_MAX):
236 self.outputs.remove(f"fgcmFitParameters{cycle}")
237 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
238 self.outputs.remove(f"fgcmZeropoints{cycle}")
239 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
240 self.outputs.remove(f"fgcmStandardStars{cycle}")
241
242 else:
243 # Multiple-cycle run
244 # Remove single-cycle outputs
245 self.outputs.remove("fgcmFitParameters")
246 self.outputs.remove("fgcmFlaggedStars")
247 self.outputs.remove("fgcmZeropoints")
248 self.outputs.remove("fgcmAtmosphereParameters")
249 self.outputs.remove("fgcmStandardStars")
250
251 # Remove outputs from cycles that are not used
252 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
253 MULTIPLE_CYCLES_MAX):
254 self.outputs.remove(f"fgcmFitParameters{cycle}")
255 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
256 self.outputs.remove(f"fgcmZeropoints{cycle}")
257 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
258 self.outputs.remove(f"fgcmStandardStars{cycle}")
259
260 # Remove non-final-cycle outputs if necessary
261 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
262 if not self.config.outputZeropointsBeforeFinalCycle:
263 self.outputs.remove(f"fgcmZeropoints{cycle}")
264 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
265 if not self.config.outputStandardsBeforeFinalCycle:
266 self.outputs.remove(f"fgcmStandardStars{cycle}")
267
268
269class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
270 pipelineConnections=FgcmFitCycleConnections):
271 """Config for FgcmFitCycle"""
272
273 doMultipleCycles = pexConfig.Field(
274 doc="Run multiple fit cycles in one task",
275 dtype=bool,
276 default=False,
277 )
278 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
279 doc=("Final cycle number in multiple cycle mode. The initial cycle "
280 "is 0, with limited parameters fit. The next cycle is 1 with "
281 "full parameter fit. The final cycle is a clean-up with no "
282 "parameters fit. There will be a total of "
283 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
284 "cycle number cannot be less than 2."),
285 dtype=int,
286 default=5,
287 min=2,
288 max=MULTIPLE_CYCLES_MAX,
289 inclusiveMax=True,
290 )
291 bands = pexConfig.ListField(
292 doc="Bands to run calibration",
293 dtype=str,
294 default=[],
295 )
296 fitBands = pexConfig.ListField(
297 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
298 "the atmosphere constrained from the 'fitBands' on the same night. "
299 "Must be a subset of `config.bands`"),
300 dtype=str,
301 default=[],
302 )
303 requiredBands = pexConfig.ListField(
304 doc=("Bands that are required for a star to be considered a calibration star. "
305 "Must be a subset of `config.bands`"),
306 dtype=str,
307 default=[],
308 )
309 physicalFilterMap = pexConfig.DictField(
310 doc="Mapping from 'physicalFilter' to band.",
311 keytype=str,
312 itemtype=str,
313 default={},
314 )
315 doReferenceCalibration = pexConfig.Field(
316 doc="Use reference catalog as additional constraint on calibration",
317 dtype=bool,
318 default=True,
319 )
320 refStarSnMin = pexConfig.Field(
321 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
322 dtype=float,
323 default=50.0,
324 )
325 refStarOutlierNSig = pexConfig.Field(
326 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
327 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
328 dtype=float,
329 default=4.0,
330 )
331 applyRefStarColorCuts = pexConfig.Field(
332 doc="Apply color cuts to reference stars?",
333 dtype=bool,
334 default=True,
335 )
336 nCore = pexConfig.Field(
337 doc="Number of cores to use",
338 dtype=int,
339 default=4,
340 )
341 nStarPerRun = pexConfig.Field(
342 doc="Number of stars to run in each chunk",
343 dtype=int,
344 default=200000,
345 )
346 nExpPerRun = pexConfig.Field(
347 doc="Number of exposures to run in each chunk",
348 dtype=int,
349 default=1000,
350 )
351 reserveFraction = pexConfig.Field(
352 doc="Fraction of stars to reserve for testing",
353 dtype=float,
354 default=0.1,
355 )
356 freezeStdAtmosphere = pexConfig.Field(
357 doc="Freeze atmosphere parameters to standard (for testing)",
358 dtype=bool,
359 default=False,
360 )
361 precomputeSuperStarInitialCycle = pexConfig.Field(
362 doc="Precompute superstar flat for initial cycle",
363 dtype=bool,
364 default=False,
365 )
366 superStarSubCcdDict = pexConfig.DictField(
367 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
368 "Must have one entry per band."),
369 keytype=str,
370 itemtype=bool,
371 default={},
372 )
373 superStarSubCcdChebyshevOrder = pexConfig.Field(
374 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
375 "Global default is first-order polynomials, and should be overridden "
376 "on a camera-by-camera basis depending on the ISR."),
377 dtype=int,
378 default=1,
379 )
380 superStarSubCcdTriangular = pexConfig.Field(
381 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
382 "suppress high-order cross terms?"),
383 dtype=bool,
384 default=False,
385 )
386 superStarSigmaClip = pexConfig.Field(
387 doc="Number of sigma to clip outliers when selecting for superstar flats",
388 dtype=float,
389 default=5.0,
390 )
391 focalPlaneSigmaClip = pexConfig.Field(
392 doc="Number of sigma to clip outliers per focal-plane.",
393 dtype=float,
394 default=4.0,
395 )
396 ccdGraySubCcdDict = pexConfig.DictField(
397 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
398 "('ccd gray') on a sub-ccd scale."),
399 keytype=str,
400 itemtype=bool,
401 default={},
402 )
403 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
404 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
405 dtype=int,
406 default=1,
407 )
408 ccdGraySubCcdTriangular = pexConfig.Field(
409 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
410 "suppress high-order cross terms?"),
411 dtype=bool,
412 default=True,
413 )
414 ccdGrayFocalPlaneDict = pexConfig.DictField(
415 doc=("Per-band specification on whether to compute focal-plane residual "
416 "('ccd gray') corrections."),
417 keytype=str,
418 itemtype=bool,
419 default={},
420 )
421 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
422 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
423 "gray corrections. If there are fewer good CCDs then the gray "
424 "correction is computed per-ccd."),
425 dtype=int,
426 default=1,
427 )
428 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
429 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
430 dtype=int,
431 default=3,
432 )
433 cycleNumber = pexConfig.Field(
434 doc=("FGCM fit cycle number. This is automatically incremented after each run "
435 "and stage of outlier rejection. See cookbook for details."),
436 dtype=int,
437 default=None,
438 )
439 isFinalCycle = pexConfig.Field(
440 doc=("Is this the final cycle of the fitting? Will automatically compute final "
441 "selection of stars and photometric exposures, and will output zeropoints "
442 "and standard stars for use in fgcmOutputProducts"),
443 dtype=bool,
444 default=False,
445 )
446 maxIterBeforeFinalCycle = pexConfig.Field(
447 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
448 "will always be 0 in the final cycle for cleanup and final selection."),
449 dtype=int,
450 default=50,
451 )
452 deltaMagBkgOffsetPercentile = pexConfig.Field(
453 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
454 "offset from local background subtraction."),
455 dtype=float,
456 default=0.25,
457 )
458 deltaMagBkgPerCcd = pexConfig.Field(
459 doc=("Compute net offset from local background subtraction per-ccd? "
460 "Otherwise, use computation per visit."),
461 dtype=bool,
462 default=False,
463 )
464 utBoundary = pexConfig.Field(
465 doc="Boundary (in UTC) from day-to-day",
466 dtype=float,
467 default=None,
468 )
469 washMjds = pexConfig.ListField(
470 doc="Mirror wash MJDs",
471 dtype=float,
472 default=(0.0,),
473 )
474 epochMjds = pexConfig.ListField(
475 doc="Epoch boundaries in MJD",
476 dtype=float,
477 default=(0.0,),
478 )
479 minObsPerBand = pexConfig.Field(
480 doc="Minimum good observations per band",
481 dtype=int,
482 default=2,
483 )
484 # TODO: When DM-16511 is done, it will be possible to get the
485 # telescope latitude directly from the camera.
486 latitude = pexConfig.Field(
487 doc="Observatory latitude",
488 dtype=float,
489 default=None,
490 )
491 defaultCameraOrientation = pexConfig.Field(
492 doc="Default camera orientation for QA plots.",
493 dtype=float,
494 default=None,
495 )
496 brightObsGrayMax = pexConfig.Field(
497 doc="Maximum gray extinction to be considered bright observation",
498 dtype=float,
499 default=0.15,
500 )
501 minStarPerCcd = pexConfig.Field(
502 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
503 "CCDs with fewer stars will have their calibration estimated from other "
504 "CCDs in the same visit, with zeropoint error increased accordingly."),
505 dtype=int,
506 default=5,
507 )
508 minCcdPerExp = pexConfig.Field(
509 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
510 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
511 dtype=int,
512 default=5,
513 )
514 maxCcdGrayErr = pexConfig.Field(
515 doc="Maximum error on CCD gray offset to be considered photometric",
516 dtype=float,
517 default=0.05,
518 )
519 minStarPerExp = pexConfig.Field(
520 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
521 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
522 dtype=int,
523 default=600,
524 )
525 minExpPerNight = pexConfig.Field(
526 doc="Minimum number of good exposures/visits to consider a partly photometric night",
527 dtype=int,
528 default=10,
529 )
530 expGrayInitialCut = pexConfig.Field(
531 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
532 "observations."),
533 dtype=float,
534 default=-0.25,
535 )
536 expGrayPhotometricCutDict = pexConfig.DictField(
537 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
538 "('gray term') for a visit to be considered photometric. Must have one "
539 "entry per band. Broad-band filters should be -0.05."),
540 keytype=str,
541 itemtype=float,
542 default={},
543 )
544 expGrayHighCutDict = pexConfig.DictField(
545 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
546 "('gray term') for a visit to be considered photometric. Must have one "
547 "entry per band. Broad-band filters should be 0.2."),
548 keytype=str,
549 itemtype=float,
550 default={},
551 )
552 expGrayRecoverCut = pexConfig.Field(
553 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
554 "Visits with more gray extinction will only get CCD zeropoints if there are "
555 "sufficient star observations (minStarPerCcd) on that CCD."),
556 dtype=float,
557 default=-1.0,
558 )
559 expVarGrayPhotometricCutDict = pexConfig.DictField(
560 doc=("Per-band specification on maximum exposure variance to be considered possibly "
561 "photometric. Must have one entry per band. Broad-band filters should be "
562 "0.0005."),
563 keytype=str,
564 itemtype=float,
565 default={},
566 )
567 expGrayErrRecoverCut = pexConfig.Field(
568 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
569 "Visits with more gray variance will only get CCD zeropoints if there are "
570 "sufficient star observations (minStarPerCcd) on that CCD."),
571 dtype=float,
572 default=0.05,
573 )
574 aperCorrFitNBins = pexConfig.Field(
575 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
576 "no fit will be performed, and the config.aperCorrInputSlopes will be "
577 "used if available."),
578 dtype=int,
579 default=10,
580 )
581 aperCorrInputSlopeDict = pexConfig.DictField(
582 doc=("Per-band specification of aperture correction input slope parameters. These "
583 "are used on the first fit iteration, and aperture correction parameters will "
584 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
585 "to set this when there is insufficient data to fit the parameters (e.g. "
586 "tract mode)."),
587 keytype=str,
588 itemtype=float,
589 default={},
590 )
591 sedboundaryterms = pexConfig.ConfigField(
592 doc="Mapping from bands to SED boundary term names used is sedterms.",
593 dtype=SedboundarytermDict,
594 )
595 sedterms = pexConfig.ConfigField(
596 doc="Mapping from terms to bands for fgcm linear SED approximations.",
597 dtype=SedtermDict,
598 )
599 sigFgcmMaxErr = pexConfig.Field(
600 doc="Maximum mag error for fitting sigma_FGCM",
601 dtype=float,
602 default=0.01,
603 )
604 sigFgcmMaxEGrayDict = pexConfig.DictField(
605 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
606 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
607 "should be 0.05."),
608 keytype=str,
609 itemtype=float,
610 default={},
611 )
612 ccdGrayMaxStarErr = pexConfig.Field(
613 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
614 "computation"),
615 dtype=float,
616 default=0.10,
617 )
618 approxThroughputDict = pexConfig.DictField(
619 doc=("Per-band specification of the approximate overall throughput at the start of "
620 "calibration observations. Must have one entry per band. Typically should "
621 "be 1.0."),
622 keytype=str,
623 itemtype=float,
624 default={},
625 )
626 sigmaCalRange = pexConfig.ListField(
627 doc="Allowed range for systematic error floor estimation",
628 dtype=float,
629 default=(0.001, 0.003),
630 )
631 sigmaCalFitPercentile = pexConfig.ListField(
632 doc="Magnitude percentile range to fit systematic error floor",
633 dtype=float,
634 default=(0.05, 0.15),
635 )
636 sigmaCalPlotPercentile = pexConfig.ListField(
637 doc="Magnitude percentile range to plot systematic error floor",
638 dtype=float,
639 default=(0.05, 0.95),
640 )
641 sigma0Phot = pexConfig.Field(
642 doc="Systematic error floor for all zeropoints",
643 dtype=float,
644 default=0.003,
645 )
646 mapLongitudeRef = pexConfig.Field(
647 doc="Reference longitude for plotting maps",
648 dtype=float,
649 default=0.0,
650 )
651 mapNSide = pexConfig.Field(
652 doc="Healpix nside for plotting maps",
653 dtype=int,
654 default=256,
655 )
656 outfileBase = pexConfig.Field(
657 doc="Filename start for plot output files",
658 dtype=str,
659 default=None,
660 )
661 starColorCuts = pexConfig.ListField(
662 doc="Encoded star-color cuts (to be cleaned up)",
663 dtype=str,
664 default=("NO_DATA",),
665 )
666 colorSplitBands = pexConfig.ListField(
667 doc="Band names to use to split stars by color. Must have 2 entries.",
668 dtype=str,
669 length=2,
670 default=('g', 'i'),
671 )
672 modelMagErrors = pexConfig.Field(
673 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
674 dtype=bool,
675 default=True,
676 )
677 useQuadraticPwv = pexConfig.Field(
678 doc="Model PWV with a quadratic term for variation through the night?",
679 dtype=bool,
680 default=False,
681 )
682 instrumentParsPerBand = pexConfig.Field(
683 doc=("Model instrumental parameters per band? "
684 "Otherwise, instrumental parameters (QE changes with time) are "
685 "shared among all bands."),
686 dtype=bool,
687 default=False,
688 )
689 instrumentSlopeMinDeltaT = pexConfig.Field(
690 doc=("Minimum time change (in days) between observations to use in constraining "
691 "instrument slope."),
692 dtype=float,
693 default=20.0,
694 )
695 fitMirrorChromaticity = pexConfig.Field(
696 doc="Fit (intraband) mirror chromatic term?",
697 dtype=bool,
698 default=False,
699 )
700 coatingMjds = pexConfig.ListField(
701 doc="Mirror coating dates in MJD",
702 dtype=float,
703 default=(0.0,),
704 )
705 outputStandardsBeforeFinalCycle = pexConfig.Field(
706 doc="Output standard stars prior to final cycle? Used in debugging.",
707 dtype=bool,
708 default=False,
709 )
710 outputZeropointsBeforeFinalCycle = pexConfig.Field(
711 doc="Output standard stars prior to final cycle? Used in debugging.",
712 dtype=bool,
713 default=False,
714 )
715 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
716 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
717 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
718 keytype=str,
719 itemtype=bool,
720 default={},
721 )
722 autoPhotometricCutNSig = pexConfig.Field(
723 doc=("Number of sigma for automatic computation of (low) photometric cut. "
724 "Cut is based on exposure gray width (per band), unless "
725 "useRepeatabilityForExpGrayCuts is set, in which case the star "
726 "repeatability is used (also per band)."),
727 dtype=float,
728 default=3.0,
729 )
730 autoHighCutNSig = pexConfig.Field(
731 doc=("Number of sigma for automatic computation of (high) outlier cut. "
732 "Cut is based on exposure gray width (per band), unless "
733 "useRepeatabilityForExpGrayCuts is set, in which case the star "
734 "repeatability is used (also per band)."),
735 dtype=float,
736 default=4.0,
737 )
738 quietMode = pexConfig.Field(
739 doc="Be less verbose with logging.",
740 dtype=bool,
741 default=False,
742 )
743 doPlots = pexConfig.Field(
744 doc="Make fgcm QA plots.",
745 dtype=bool,
746 default=True,
747 )
748 randomSeed = pexConfig.Field(
749 doc="Random seed for fgcm for consistency in tests.",
750 dtype=int,
751 default=None,
752 optional=True,
753 )
754 deltaAperFitMinNgoodObs = pexConfig.Field(
755 doc="Minimum number of good observations to use mean delta-aper values in fits.",
756 dtype=int,
757 default=2,
758 )
759 deltaAperFitPerCcdNx = pexConfig.Field(
760 doc=("Number of x bins per ccd when computing delta-aper background offsets. "
761 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
762 dtype=int,
763 default=10,
764 )
765 deltaAperFitPerCcdNy = pexConfig.Field(
766 doc=("Number of y bins per ccd when computing delta-aper background offsets. "
767 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
768 dtype=int,
769 default=10,
770 )
771 deltaAperFitSpatialNside = pexConfig.Field(
772 doc="Healpix nside to compute spatial delta-aper background offset maps.",
773 dtype=int,
774 default=64,
775 )
776 deltaAperInnerRadiusArcsec = pexConfig.Field(
777 doc=("Inner radius used to compute deltaMagAper (arcseconds). "
778 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if "
779 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
780 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
781 dtype=float,
782 default=0.0,
783 )
784 deltaAperOuterRadiusArcsec = pexConfig.Field(
785 doc=("Outer radius used to compute deltaMagAper (arcseconds). "
786 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if "
787 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
788 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
789 dtype=float,
790 default=0.0,
791 )
792 doComputeDeltaAperPerVisit = pexConfig.Field(
793 doc=("Do the computation of delta-aper background offsets per visit? "
794 "Note: this option can be very slow when there are many visits."),
795 dtype=bool,
796 default=False,
797 )
798 doComputeDeltaAperPerStar = pexConfig.Field(
799 doc="Do the computation of delta-aper mean values per star?",
800 dtype=bool,
801 default=True,
802 )
803 doComputeDeltaAperMap = pexConfig.Field(
804 doc=("Do the computation of delta-aper spatial maps? "
805 "This is only used if ``doComputeDeltaAperPerStar`` is True,"),
806 dtype=bool,
807 default=False,
808 )
809 doComputeDeltaAperPerCcd = pexConfig.Field(
810 doc="Do the computation of per-ccd delta-aper background offsets?",
811 dtype=bool,
812 default=False,
813 )
814
815 def validate(self):
816 super().validate()
817
818 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
819 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
820 raise RuntimeError(msg)
821 if self.connections.cycleNumber != str(self.cycleNumber):
822 msg = "cycleNumber in template must be equal to connections.cycleNumber"
823 raise RuntimeError(msg)
824
825 for band in self.fitBands:
826 if band not in self.bands:
827 msg = 'fitBand %s not in bands' % (band)
828 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
829 for band in self.requiredBands:
830 if band not in self.bands:
831 msg = 'requiredBand %s not in bands' % (band)
832 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
833 for band in self.colorSplitBands:
834 if band not in self.bands:
835 msg = 'colorSplitBand %s not in bands' % (band)
836 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
837 for band in self.bands:
838 if band not in self.superStarSubCcdDict:
839 msg = 'band %s not in superStarSubCcdDict' % (band)
840 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
841 self, msg)
842 if band not in self.ccdGraySubCcdDict:
843 msg = 'band %s not in ccdGraySubCcdDict' % (band)
844 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
845 self, msg)
846 if band not in self.expGrayPhotometricCutDict:
847 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
848 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
849 self, msg)
850 if band not in self.expGrayHighCutDict:
851 msg = 'band %s not in expGrayHighCutDict' % (band)
852 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
853 self, msg)
854 if band not in self.expVarGrayPhotometricCutDict:
855 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
856 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
857 self, msg)
858 if band not in self.sigFgcmMaxEGrayDict:
859 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
860 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
861 self, msg)
862 if band not in self.approxThroughputDict:
863 msg = 'band %s not in approxThroughputDict' % (band)
864 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
865 self, msg)
866 if band not in self.useRepeatabilityForExpGrayCutsDict:
867 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
868 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
869 self, msg)
870
871 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \
872 or self.doComputeDeltaAperPerCcd:
873 if self.deltaAperInnerRadiusArcsec <= 0.0:
874 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.'
875 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec,
876 self, msg)
877 if self.deltaAperOuterRadiusArcsec <= 0.0:
878 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.'
879 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
880 self, msg)
881 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec:
882 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if '
883 'deltaAper computations are turned on.')
884 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
885 self, msg)
886
887
888class FgcmFitCycleTask(pipeBase.PipelineTask):
889 """
890 Run Single fit cycle for FGCM global calibration
891 """
892
893 ConfigClass = FgcmFitCycleConfig
894 _DefaultName = "fgcmFitCycle"
895
896 def __init__(self, initInputs=None, **kwargs):
897 super().__init__(**kwargs)
898
899 def runQuantum(self, butlerQC, inputRefs, outputRefs):
900 camera = butlerQC.get(inputRefs.camera)
901
902 handleDict = {}
903
904 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
905 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
906 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
907 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
908 handleDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
909 if self.config.doReferenceCalibration:
910 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
911 if self.config.cycleNumber > 0:
912 handleDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
913 handleDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
914
915 fgcmDatasetDict = None
916 if self.config.doMultipleCycles:
917 # Run multiple cycles at once.
918 config = copy.copy(self.config)
919 config.update(cycleNumber=0)
920 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
921 if cycle == self.config.multipleCyclesFinalCycleNumber:
922 config.update(isFinalCycle=True)
923
924 if cycle > 0:
925 handleDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
926 handleDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
927
928 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config)
929 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
930 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
931 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
932 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
933 if self.outputZeropoints:
934 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
935 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
936 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
937 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
938 if self.outputStandards:
939 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
940 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
941 else:
942 # Run a single cycle
943 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict)
944
945 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
946 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
947 if self.outputZeropoints:
948 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
949 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
950 if self.outputStandards:
951 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
952
953 def _fgcmFitCycle(self, camera, handleDict, config=None):
954 """
955 Run the fit cycle
956
957 Parameters
958 ----------
959 camera : `lsst.afw.cameraGeom.Camera`
960 handleDict : `dict`
961 All handles are `lsst.daf.butler.DeferredDatasetHandle`
962 handle dictionary with keys:
963
964 ``"fgcmLookUpTable"``
965 handle for the FGCM look-up table.
966 ``"fgcmVisitCatalog"``
967 handle for visit summary catalog.
968 ``"fgcmStarObservations"``
969 handle for star observation catalog.
970 ``"fgcmStarIds"``
971 handle for star id catalog.
972 ``"fgcmStarIndices"``
973 handle for star index catalog.
974 ``"fgcmReferenceStars"``
975 handle for matched reference star catalog.
976 ``"fgcmFlaggedStars"``
977 handle for flagged star catalog.
978 ``"fgcmFitParameters"``
979 handle for fit parameter catalog.
980 config : `lsst.pex.config.Config`, optional
981 Configuration to use to override self.config.
982
983 Returns
984 -------
985 fgcmDatasetDict : `dict`
986 Dictionary of datasets to persist.
987 """
988 if config is not None:
989 _config = config
990 else:
991 _config = self.config
992
993 # Set defaults on whether to output standards and zeropoints
994 self.maxIter = _config.maxIterBeforeFinalCycle
995 self.outputStandards = _config.outputStandardsBeforeFinalCycle
996 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
997 self.resetFitParameters = True
998
999 if _config.isFinalCycle:
1000 # This is the final fit cycle, so we do not want to reset fit
1001 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1002 # and we always want to output standards and zeropoints
1003 self.maxIter = 0
1004 self.outputStandards = True
1005 self.outputZeropoints = True
1006 self.resetFitParameters = False
1007
1008 lutCat = handleDict['fgcmLookUpTable'].get()
1009 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1010 dict(_config.physicalFilterMap))
1011 del lutCat
1012
1013 configDict = makeConfigDict(_config, self.log, camera,
1014 self.maxIter, self.resetFitParameters,
1015 self.outputZeropoints,
1016 lutIndexVals[0]['FILTERNAMES'])
1017
1018 # next we need the exposure/visit information
1019 visitCat = handleDict['fgcmVisitCatalog'].get()
1020 fgcmExpInfo = translateVisitCatalog(visitCat)
1021 del visitCat
1022
1023 focalPlaneProjector = FocalPlaneProjector(camera,
1024 self.config.defaultCameraOrientation)
1025
1026 noFitsDict = {'lutIndex': lutIndexVals,
1027 'lutStd': lutStd,
1028 'expInfo': fgcmExpInfo,
1029 'focalPlaneProjector': focalPlaneProjector}
1030
1031 # set up the fitter object
1032 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1033 noFitsDict=noFitsDict, noOutput=True)
1034
1035 # create the parameter object
1036 if (fgcmFitCycle.initialCycle):
1037 # cycle = 0, initial cycle
1038 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1039 fgcmLut,
1040 fgcmExpInfo)
1041 else:
1042 if isinstance(handleDict['fgcmFitParameters'], afwTable.BaseCatalog):
1043 parCat = handleDict['fgcmFitParameters']
1044 else:
1045 parCat = handleDict['fgcmFitParameters'].get()
1046 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1047 del parCat
1048 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1049 fgcmExpInfo,
1050 inParInfo,
1051 inParams,
1052 inSuperStar)
1053
1054 # set up the stars...
1055 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1056
1057 starObs = handleDict['fgcmStarObservations'].get()
1058 starIds = handleDict['fgcmStarIds'].get()
1059 starIndices = handleDict['fgcmStarIndices'].get()
1060
1061 # grab the flagged stars if available
1062 if 'fgcmFlaggedStars' in handleDict:
1063 if isinstance(handleDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1064 flaggedStars = handleDict['fgcmFlaggedStars']
1065 else:
1066 flaggedStars = handleDict['fgcmFlaggedStars'].get()
1067 flagId = flaggedStars['objId'][:]
1068 flagFlag = flaggedStars['objFlag'][:]
1069 else:
1070 flaggedStars = None
1071 flagId = None
1072 flagFlag = None
1073
1074 if _config.doReferenceCalibration:
1075 refStars = handleDict['fgcmReferenceStars'].get()
1076
1077 refMag, refMagErr = extractReferenceMags(refStars,
1078 _config.bands,
1079 _config.physicalFilterMap)
1080 refId = refStars['fgcm_id'][:]
1081 else:
1082 refStars = None
1083 refId = None
1084 refMag = None
1085 refMagErr = None
1086
1087 # match star observations to visits
1088 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1089 # actually be transferred into fgcm using the indexing below.
1090 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1091
1092 # The fgcmStars.loadStars method will copy all the star information into
1093 # special shared memory objects that will not blow up the memory usage when
1094 # used with python multiprocessing. Once all the numbers are copied,
1095 # it is necessary to release all references to the objects that previously
1096 # stored the data to ensure that the garbage collector can clear the memory,
1097 # and ensure that this memory is not copied when multiprocessing kicks in.
1098
1099 # We determine the conversion from the native units (typically radians) to
1100 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1101 # numpy arrays rather than Angles, which would we approximately 600x slower.
1102 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1103
1104 fgcmStars.loadStars(fgcmPars,
1105 starObs['visit'][starIndices['obsIndex']],
1106 starObs['ccd'][starIndices['obsIndex']],
1107 starObs['ra'][starIndices['obsIndex']] * conv,
1108 starObs['dec'][starIndices['obsIndex']] * conv,
1109 starObs['instMag'][starIndices['obsIndex']],
1110 starObs['instMagErr'][starIndices['obsIndex']],
1111 fgcmExpInfo['FILTERNAME'][visitIndex],
1112 starIds['fgcm_id'][:],
1113 starIds['ra'][:],
1114 starIds['dec'][:],
1115 starIds['obsArrIndex'][:],
1116 starIds['nObs'][:],
1117 obsX=starObs['x'][starIndices['obsIndex']],
1118 obsY=starObs['y'][starIndices['obsIndex']],
1119 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1120 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']],
1121 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1122 refID=refId,
1123 refMag=refMag,
1124 refMagErr=refMagErr,
1125 flagID=flagId,
1126 flagFlag=flagFlag,
1127 computeNobs=True)
1128
1129 # Release all references to temporary objects holding star data (see above)
1130 del starObs
1131 del starIds
1132 del starIndices
1133 del flagId
1134 del flagFlag
1135 del flaggedStars
1136 del refStars
1137 del refId
1138 del refMag
1139 del refMagErr
1140
1141 # and set the bits in the cycle object
1142 fgcmFitCycle.setLUT(fgcmLut)
1143 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1144 fgcmFitCycle.setPars(fgcmPars)
1145
1146 # finish the setup
1147 fgcmFitCycle.finishSetup()
1148
1149 # and run
1150 fgcmFitCycle.run()
1151
1152
1155
1156 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1157
1158 # Output the config for the next cycle
1159 # We need to make a copy since the input one has been frozen
1160
1161 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1162 i, b in enumerate(_config.bands)}
1163 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1164 i, band in enumerate(_config.bands)}
1165
1166 outConfig = copy.copy(_config)
1167 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1168 precomputeSuperStarInitialCycle=False,
1169 freezeStdAtmosphere=False,
1170 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1171 expGrayHighCutDict=updatedHighCutDict)
1172
1173 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1174 cycleNumber=str(_config.cycleNumber + 1))
1175
1176 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1177 outConfig.cycleNumber)
1178 outConfig.save(configFileName)
1179
1180 if _config.isFinalCycle == 1:
1181 # We are done, ready to output products
1182 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1183 else:
1184 self.log.info("Saved config for next cycle to %s" % (configFileName))
1185 self.log.info("Be sure to look at:")
1186 self.log.info(" config.expGrayPhotometricCut")
1187 self.log.info(" config.expGrayHighCut")
1188 self.log.info("If you are satisfied with the fit, please set:")
1189 self.log.info(" config.isFinalCycle = True")
1190
1191 fgcmFitCycle.freeSharedMemory()
1192
1193 return fgcmDatasetDict, outConfig
1194
1195 def _loadParameters(self, parCat):
1196 """
1197 Load FGCM parameters from a previous fit cycle
1198
1199 Parameters
1200 ----------
1201 parCat : `lsst.afw.table.BaseCatalog`
1202 Parameter catalog in afw table form.
1203
1204 Returns
1205 -------
1206 inParInfo: `numpy.ndarray`
1207 Numpy array parameter information formatted for input to fgcm
1208 inParameters: `numpy.ndarray`
1209 Numpy array parameter values formatted for input to fgcm
1210 inSuperStar: `numpy.array`
1211 Superstar flat formatted for input to fgcm
1212 """
1213 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1214 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1215
1216 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1217 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1218 (parLutFilterNames.size, )),
1219 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1220 ('LNTAUUNIT', 'f8'),
1221 ('LNTAUSLOPEUNIT', 'f8'),
1222 ('ALPHAUNIT', 'f8'),
1223 ('LNPWVUNIT', 'f8'),
1224 ('LNPWVSLOPEUNIT', 'f8'),
1225 ('LNPWVQUADRATICUNIT', 'f8'),
1226 ('LNPWVGLOBALUNIT', 'f8'),
1227 ('O3UNIT', 'f8'),
1228 ('QESYSUNIT', 'f8'),
1229 ('FILTEROFFSETUNIT', 'f8'),
1230 ('HASEXTERNALPWV', 'i2'),
1231 ('HASEXTERNALTAU', 'i2')])
1232 inParInfo['NCCD'] = parCat['nCcd']
1233 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1234 inParInfo['FITBANDS'][:] = parFitBands
1235 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1236 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1237
1238 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1239 ('PARO3', 'f8', (parCat['parO3'].size, )),
1240 ('PARLNTAUINTERCEPT', 'f8',
1241 (parCat['parLnTauIntercept'].size, )),
1242 ('PARLNTAUSLOPE', 'f8',
1243 (parCat['parLnTauSlope'].size, )),
1244 ('PARLNPWVINTERCEPT', 'f8',
1245 (parCat['parLnPwvIntercept'].size, )),
1246 ('PARLNPWVSLOPE', 'f8',
1247 (parCat['parLnPwvSlope'].size, )),
1248 ('PARLNPWVQUADRATIC', 'f8',
1249 (parCat['parLnPwvQuadratic'].size, )),
1250 ('PARQESYSINTERCEPT', 'f8',
1251 (parCat['parQeSysIntercept'].size, )),
1252 ('COMPQESYSSLOPE', 'f8',
1253 (parCat['compQeSysSlope'].size, )),
1254 ('PARFILTEROFFSET', 'f8',
1255 (parCat['parFilterOffset'].size, )),
1256 ('PARFILTEROFFSETFITFLAG', 'i2',
1257 (parCat['parFilterOffsetFitFlag'].size, )),
1258 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1259 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1260 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1261 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1262 ('COMPABSTHROUGHPUT', 'f8',
1263 (parCat['compAbsThroughput'].size, )),
1264 ('COMPREFOFFSET', 'f8',
1265 (parCat['compRefOffset'].size, )),
1266 ('COMPREFSIGMA', 'f8',
1267 (parCat['compRefSigma'].size, )),
1268 ('COMPMIRRORCHROMATICITY', 'f8',
1269 (parCat['compMirrorChromaticity'].size, )),
1270 ('MIRRORCHROMATICITYPIVOT', 'f8',
1271 (parCat['mirrorChromaticityPivot'].size, )),
1272 ('COMPMEDIANSEDSLOPE', 'f8',
1273 (parCat['compMedianSedSlope'].size, )),
1274 ('COMPAPERCORRPIVOT', 'f8',
1275 (parCat['compAperCorrPivot'].size, )),
1276 ('COMPAPERCORRSLOPE', 'f8',
1277 (parCat['compAperCorrSlope'].size, )),
1278 ('COMPAPERCORRSLOPEERR', 'f8',
1279 (parCat['compAperCorrSlopeErr'].size, )),
1280 ('COMPAPERCORRRANGE', 'f8',
1281 (parCat['compAperCorrRange'].size, )),
1282 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1283 (parCat['compModelErrExptimePivot'].size, )),
1284 ('COMPMODELERRFWHMPIVOT', 'f8',
1285 (parCat['compModelErrFwhmPivot'].size, )),
1286 ('COMPMODELERRSKYPIVOT', 'f8',
1287 (parCat['compModelErrSkyPivot'].size, )),
1288 ('COMPMODELERRPARS', 'f8',
1289 (parCat['compModelErrPars'].size, )),
1290 ('COMPEXPGRAY', 'f8',
1291 (parCat['compExpGray'].size, )),
1292 ('COMPVARGRAY', 'f8',
1293 (parCat['compVarGray'].size, )),
1294 ('COMPEXPDELTAMAGBKG', 'f8',
1295 (parCat['compExpDeltaMagBkg'].size, )),
1296 ('COMPNGOODSTARPEREXP', 'i4',
1297 (parCat['compNGoodStarPerExp'].size, )),
1298 ('COMPSIGFGCM', 'f8',
1299 (parCat['compSigFgcm'].size, )),
1300 ('COMPSIGMACAL', 'f8',
1301 (parCat['compSigmaCal'].size, )),
1302 ('COMPRETRIEVEDLNPWV', 'f8',
1303 (parCat['compRetrievedLnPwv'].size, )),
1304 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1305 (parCat['compRetrievedLnPwvRaw'].size, )),
1306 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1307 (parCat['compRetrievedLnPwvFlag'].size, )),
1308 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1309 (parCat['compRetrievedTauNight'].size, )),
1310 ('COMPEPSILON', 'f8',
1311 (parCat['compEpsilon'].size, )),
1312 ('COMPMEDDELTAAPER', 'f8',
1313 (parCat['compMedDeltaAper'].size, )),
1314 ('COMPGLOBALEPSILON', 'f4',
1315 (parCat['compGlobalEpsilon'].size, )),
1316 ('COMPEPSILONMAP', 'f4',
1317 (parCat['compEpsilonMap'].size, )),
1318 ('COMPEPSILONNSTARMAP', 'i4',
1319 (parCat['compEpsilonNStarMap'].size, )),
1320 ('COMPEPSILONCCDMAP', 'f4',
1321 (parCat['compEpsilonCcdMap'].size, )),
1322 ('COMPEPSILONCCDNSTARMAP', 'i4',
1323 (parCat['compEpsilonCcdNStarMap'].size, ))])
1324
1325 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1326 inParams['PARO3'][:] = parCat['parO3'][0, :]
1327 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1328 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1329 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1330 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1331 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1332 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1333 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1334 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1335 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1336 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1337 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1338 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1339 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1340 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1341 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1342 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1343 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1344 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1345 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1346 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1347 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1348 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1349 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1350 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1351 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1352 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1353 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1354 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1355 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1356 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1357 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1358 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1359 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1360 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1361 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1362 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1363 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :]
1364 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :]
1365 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :]
1366 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :]
1367 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :]
1368 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :]
1369 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :]
1370
1371 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1372 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1373
1374 return (inParInfo, inParams, inSuperStar)
1375
1376 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1377 """
1378 Persist FGCM datasets through the butler.
1379
1380 Parameters
1381 ----------
1382 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1383 Fgcm Fit cycle object
1384 """
1385 fgcmDatasetDict = {}
1386
1387 # Save the parameters
1388 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1389
1390 parSchema = afwTable.Schema()
1391
1392 comma = ','
1393 lutFilterNameString = comma.join([n.decode('utf-8')
1394 for n in parInfo['LUTFILTERNAMES'][0]])
1395 fitBandString = comma.join([n.decode('utf-8')
1396 for n in parInfo['FITBANDS'][0]])
1397
1398 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1399 lutFilterNameString, fitBandString)
1400 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1401 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1402 lutFilterNameString, fitBandString)
1403
1404 fgcmDatasetDict['fgcmFitParameters'] = parCat
1405
1406 # Save the indices of the flagged stars
1407 # (stars that have been (a) reserved from the fit for testing and
1408 # (b) bad stars that have failed quality checks.)
1409 flagStarSchema = self._makeFlagStarSchema()
1410 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1411 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1412
1413 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1414
1415 # Save the zeropoint information and atmospheres only if desired
1416 if self.outputZeropoints:
1417 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1418 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1419
1420 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1421 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1422
1423 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1424
1425 # Save atmosphere values
1426 # These are generated by the same code that generates zeropoints
1427 atmSchema = makeAtmSchema()
1428 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1429
1430 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1431
1432 # Save the standard stars (if configured)
1433 if self.outputStandards:
1434 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1435 stdSchema = makeStdSchema(len(goodBands))
1436 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1437
1438 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1439
1440 return fgcmDatasetDict
1441
1442 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1443 lutFilterNameString, fitBandString):
1444 """
1445 Make the parameter persistence schema
1446
1447 Parameters
1448 ----------
1449 parInfo: `numpy.ndarray`
1450 Parameter information returned by fgcm
1451 pars: `numpy.ndarray`
1452 Parameter values returned by fgcm
1453 parSuperStarFlat: `numpy.array`
1454 Superstar flat values returned by fgcm
1455 lutFilterNameString: `str`
1456 Combined string of all the lutFilterNames
1457 fitBandString: `str`
1458 Combined string of all the fitBands
1459
1460 Returns
1461 -------
1462 parSchema: `afwTable.schema`
1463 """
1464
1465 parSchema = afwTable.Schema()
1466
1467 # parameter info section
1468 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1469 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1470 size=len(lutFilterNameString))
1471 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1472 size=len(fitBandString))
1473 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1474 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1475 doc='Step units for ln(AOD) slope')
1476 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1477 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1478 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1479 doc='Step units for ln(pwv) slope')
1480 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1481 doc='Step units for ln(pwv) quadratic term')
1482 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1483 doc='Step units for global ln(pwv) parameters')
1484 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1485 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1486 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1487 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1488 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1489
1490 # parameter section
1491 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1492 size=pars['PARALPHA'].size)
1493 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1494 size=pars['PARO3'].size)
1495 parSchema.addField('parLnTauIntercept', type='ArrayD',
1496 doc='ln(Tau) intercept parameter vector',
1497 size=pars['PARLNTAUINTERCEPT'].size)
1498 parSchema.addField('parLnTauSlope', type='ArrayD',
1499 doc='ln(Tau) slope parameter vector',
1500 size=pars['PARLNTAUSLOPE'].size)
1501 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1502 size=pars['PARLNPWVINTERCEPT'].size)
1503 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1504 size=pars['PARLNPWVSLOPE'].size)
1505 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1506 size=pars['PARLNPWVQUADRATIC'].size)
1507 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1508 size=pars['PARQESYSINTERCEPT'].size)
1509 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1510 size=pars[0]['COMPQESYSSLOPE'].size)
1511 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1512 size=pars['PARFILTEROFFSET'].size)
1513 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1514 size=pars['PARFILTEROFFSETFITFLAG'].size)
1515 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1516 doc='Global scale for retrieved ln(pwv)')
1517 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1518 doc='Global offset for retrieved ln(pwv)')
1519 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1520 doc='Nightly offset for retrieved ln(pwv)',
1521 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1522 parSchema.addField('compAbsThroughput', type='ArrayD',
1523 doc='Absolute throughput (relative to transmission curves)',
1524 size=pars['COMPABSTHROUGHPUT'].size)
1525 parSchema.addField('compRefOffset', type='ArrayD',
1526 doc='Offset between reference stars and calibrated stars',
1527 size=pars['COMPREFOFFSET'].size)
1528 parSchema.addField('compRefSigma', type='ArrayD',
1529 doc='Width of reference star/calibrated star distribution',
1530 size=pars['COMPREFSIGMA'].size)
1531 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1532 doc='Computed mirror chromaticity terms',
1533 size=pars['COMPMIRRORCHROMATICITY'].size)
1534 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1535 doc='Mirror chromaticity pivot mjd',
1536 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1537 parSchema.addField('compMedianSedSlope', type='ArrayD',
1538 doc='Computed median SED slope (per band)',
1539 size=pars['COMPMEDIANSEDSLOPE'].size)
1540 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1541 size=pars['COMPAPERCORRPIVOT'].size)
1542 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1543 size=pars['COMPAPERCORRSLOPE'].size)
1544 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1545 size=pars['COMPAPERCORRSLOPEERR'].size)
1546 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1547 size=pars['COMPAPERCORRRANGE'].size)
1548 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1549 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1550 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1551 size=pars['COMPMODELERRFWHMPIVOT'].size)
1552 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1553 size=pars['COMPMODELERRSKYPIVOT'].size)
1554 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1555 size=pars['COMPMODELERRPARS'].size)
1556 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1557 size=pars['COMPEXPGRAY'].size)
1558 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1559 size=pars['COMPVARGRAY'].size)
1560 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1561 doc='Computed exposure offset due to background',
1562 size=pars['COMPEXPDELTAMAGBKG'].size)
1563 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1564 doc='Computed number of good stars per exposure',
1565 size=pars['COMPNGOODSTARPEREXP'].size)
1566 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1567 size=pars['COMPSIGFGCM'].size)
1568 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1569 size=pars['COMPSIGMACAL'].size)
1570 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1571 size=pars['COMPRETRIEVEDLNPWV'].size)
1572 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1573 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1574 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1575 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1576 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1577 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1578 parSchema.addField('compEpsilon', type='ArrayD',
1579 doc='Computed epsilon background offset per visit (nJy/arcsec2)',
1580 size=pars['COMPEPSILON'].size)
1581 parSchema.addField('compMedDeltaAper', type='ArrayD',
1582 doc='Median delta mag aper per visit',
1583 size=pars['COMPMEDDELTAAPER'].size)
1584 parSchema.addField('compGlobalEpsilon', type='ArrayD',
1585 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)',
1586 size=pars['COMPGLOBALEPSILON'].size)
1587 parSchema.addField('compEpsilonMap', type='ArrayD',
1588 doc='Computed epsilon maps (nJy/arcsec2)',
1589 size=pars['COMPEPSILONMAP'].size)
1590 parSchema.addField('compEpsilonNStarMap', type='ArrayI',
1591 doc='Number of stars per pixel in computed epsilon maps',
1592 size=pars['COMPEPSILONNSTARMAP'].size)
1593 parSchema.addField('compEpsilonCcdMap', type='ArrayD',
1594 doc='Computed epsilon ccd maps (nJy/arcsec2)',
1595 size=pars['COMPEPSILONCCDMAP'].size)
1596 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI',
1597 doc='Number of stars per ccd bin in epsilon ccd maps',
1598 size=pars['COMPEPSILONCCDNSTARMAP'].size)
1599 # superstarflat section
1600 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1601 size=4)
1602 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1603 size=parSuperStarFlat.size)
1604
1605 return parSchema
1606
1607 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1608 lutFilterNameString, fitBandString):
1609 """
1610 Make the FGCM parameter catalog for persistence
1611
1612 Parameters
1613 ----------
1614 parSchema: `lsst.afw.table.Schema`
1615 Parameter catalog schema
1616 pars: `numpy.ndarray`
1617 FGCM parameters to put into parCat
1618 parSuperStarFlat: `numpy.array`
1619 FGCM superstar flat array to put into parCat
1620 lutFilterNameString: `str`
1621 Combined string of all the lutFilterNames
1622 fitBandString: `str`
1623 Combined string of all the fitBands
1624
1625 Returns
1626 -------
1627 parCat: `afwTable.BasicCatalog`
1628 Atmosphere and instrumental model parameter catalog for persistence
1629 """
1630
1631 parCat = afwTable.BaseCatalog(parSchema)
1632 parCat.reserve(1)
1633
1634 # The parameter catalog just has one row, with many columns for all the
1635 # atmosphere and instrument fit parameters
1636 rec = parCat.addNew()
1637
1638 # info section
1639 rec['nCcd'] = parInfo['NCCD']
1640 rec['lutFilterNames'] = lutFilterNameString
1641 rec['fitBands'] = fitBandString
1642 # note these are not currently supported here.
1643 rec['hasExternalPwv'] = 0
1644 rec['hasExternalTau'] = 0
1645
1646 # parameter section
1647
1648 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1649
1650 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1651 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1652 'parQeSysIntercept', 'compQeSysSlope',
1653 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1654 'parFilterOffset', 'parFilterOffsetFitFlag',
1655 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1656 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1657 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1658 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1659 'compModelErrSkyPivot', 'compModelErrPars',
1660 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1661 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1662 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1663 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper',
1664 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap',
1665 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap']
1666
1667 for scalarName in scalarNames:
1668 rec[scalarName] = pars[scalarName.upper()]
1669
1670 for arrName in arrNames:
1671 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1672
1673 # superstar section
1674 rec['superstarSize'][:] = parSuperStarFlat.shape
1675 rec['superstar'][:] = parSuperStarFlat.ravel()
1676
1677 return parCat
1678
1679 def _makeFlagStarSchema(self):
1680 """
1681 Make the flagged-stars schema
1682
1683 Returns
1684 -------
1685 flagStarSchema: `lsst.afw.table.Schema`
1686 """
1687
1688 flagStarSchema = afwTable.Schema()
1689
1690 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1691 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1692
1693 return flagStarSchema
1694
1695 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1696 """
1697 Make the flagged star catalog for persistence
1698
1699 Parameters
1700 ----------
1701 flagStarSchema: `lsst.afw.table.Schema`
1702 Flagged star schema
1703 flagStarStruct: `numpy.ndarray`
1704 Flagged star structure from fgcm
1705
1706 Returns
1707 -------
1708 flagStarCat: `lsst.afw.table.BaseCatalog`
1709 Flagged star catalog for persistence
1710 """
1711
1712 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1713 flagStarCat.resize(flagStarStruct.size)
1714
1715 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1716 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1717
1718 return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:855
def makeStdSchema(nBands)
Definition: utilities.py:751
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:718
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:51
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:229
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:639
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:786
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:553
def translateVisitCatalog(visitCat)
Definition: utilities.py:359