Coverage for python/lsst/fgcmcal/fgcmFitCycle.py: 21%
574 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-19 13:18 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-19 13:18 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import copy
38import numpy as np
40import lsst.pex.config as pexConfig
41import lsst.pipe.base as pipeBase
42from lsst.pipe.base import connectionTypes
43import lsst.afw.table as afwTable
45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
46from .utilities import extractReferenceMags
47from .utilities import makeZptSchema, makeZptCat
48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
49from .sedterms import SedboundarytermDict, SedtermDict
50from .focalPlaneProjector import FocalPlaneProjector
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask']
56MULTIPLE_CYCLES_MAX = 10
59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument",),
61 defaultTemplates={"previousCycleNumber": "-1",
62 "cycleNumber": "0"}):
63 camera = connectionTypes.PrerequisiteInput(
64 doc="Camera instrument",
65 name="camera",
66 storageClass="Camera",
67 dimensions=("instrument",),
68 isCalibration=True,
69 )
71 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
72 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
73 "chromatic corrections."),
74 name="fgcmLookUpTable",
75 storageClass="Catalog",
76 dimensions=("instrument",),
77 deferLoad=True,
78 )
80 fgcmVisitCatalog = connectionTypes.Input(
81 doc="Catalog of visit information for fgcm",
82 name="fgcmVisitCatalog",
83 storageClass="Catalog",
84 dimensions=("instrument",),
85 deferLoad=True,
86 )
88 fgcmStarObservationsParquet = connectionTypes.Input(
89 doc=("Catalog of star observations for fgcm, in parquet format. "
90 "Used if useParquetCatalogFormat is True."),
91 name="fgcm_star_observations",
92 storageClass="ArrowAstropy",
93 dimensions=("instrument",),
94 deferLoad=True,
95 )
97 fgcmStarIdsParquet = connectionTypes.Input(
98 doc=("Catalog of fgcm calibration star IDs, in parquet format. "
99 "Used if useParquetCatalogFormat is True."),
100 name="fgcm_star_ids",
101 storageClass="ArrowAstropy",
102 dimensions=("instrument",),
103 deferLoad=True,
104 )
106 fgcmReferenceStarsParquet = connectionTypes.Input(
107 doc=("Catalog of fgcm-matched reference stars, in parquet format. "
108 "Used if useParquetCatalogFormat is True."),
109 name="fgcm_reference_stars",
110 storageClass="ArrowAstropy",
111 dimensions=("instrument",),
112 deferLoad=True,
113 )
115 fgcmStarObservations = connectionTypes.Input(
116 doc=("Catalog of star observations for fgcm; old format. "
117 "Used if useParquetCatalogFormat is False."),
118 name="fgcmStarObservations",
119 storageClass="Catalog",
120 dimensions=("instrument",),
121 deferLoad=True,
122 )
124 fgcmStarIds = connectionTypes.Input(
125 doc=("Catalog of fgcm calibration star IDs. "
126 "Used if useParquetCatalogFormat is False."),
127 name="fgcmStarIds",
128 storageClass="Catalog",
129 dimensions=("instrument",),
130 deferLoad=True,
131 )
133 fgcmStarIndices = connectionTypes.Input(
134 doc=("Catalog of fgcm calibration star indices; old format."
135 "Used if useParquetCatalogFormat is False."),
136 name="fgcmStarIndices",
137 storageClass="Catalog",
138 dimensions=("instrument",),
139 deferLoad=True,
140 )
142 fgcmReferenceStars = connectionTypes.Input(
143 doc=("Catalog of fgcm-matched reference stars; old format."
144 "Used if useParquetCatalogFormat is False."),
145 name="fgcmReferenceStars",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 deferLoad=True,
149 )
151 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
152 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
153 name="fgcmFlaggedStars{previousCycleNumber}",
154 storageClass="Catalog",
155 dimensions=("instrument",),
156 deferLoad=True,
157 )
159 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
160 doc="Catalog of fgcm fit parameters from previous fit cycle",
161 name="fgcmFitParameters{previousCycleNumber}",
162 storageClass="Catalog",
163 dimensions=("instrument",),
164 deferLoad=True,
165 )
167 fgcmFitParameters = connectionTypes.Output(
168 doc="Catalog of fgcm fit parameters from current fit cycle",
169 name="fgcmFitParameters{cycleNumber}",
170 storageClass="Catalog",
171 dimensions=("instrument",),
172 )
174 fgcmFlaggedStars = connectionTypes.Output(
175 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
176 name="fgcmFlaggedStars{cycleNumber}",
177 storageClass="Catalog",
178 dimensions=("instrument",),
179 )
181 fgcmZeropoints = connectionTypes.Output(
182 doc="Catalog of fgcm zeropoint data from current fit cycle",
183 name="fgcmZeropoints{cycleNumber}",
184 storageClass="Catalog",
185 dimensions=("instrument",),
186 )
188 fgcmAtmosphereParameters = connectionTypes.Output(
189 doc="Catalog of atmospheric fit parameters from current fit cycle",
190 name="fgcmAtmosphereParameters{cycleNumber}",
191 storageClass="Catalog",
192 dimensions=("instrument",),
193 )
195 fgcmStandardStars = connectionTypes.Output(
196 doc="Catalog of standard star magnitudes from current fit cycle",
197 name="fgcmStandardStars{cycleNumber}",
198 storageClass="SimpleCatalog",
199 dimensions=("instrument",),
200 )
202 # Add connections for running multiple cycles
203 # This uses vars() to alter the class __dict__ to programmatically
204 # write many similar outputs.
205 for cycle in range(MULTIPLE_CYCLES_MAX):
206 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
207 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
208 name=f"fgcmFitParameters{cycle}",
209 storageClass="Catalog",
210 dimensions=("instrument",),
211 )
212 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
213 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
214 name=f"fgcmFlaggedStars{cycle}",
215 storageClass="Catalog",
216 dimensions=("instrument",),
217 )
218 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
219 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
220 name=f"fgcmZeropoints{cycle}",
221 storageClass="Catalog",
222 dimensions=("instrument",),
223 )
224 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
225 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
226 name=f"fgcmAtmosphereParameters{cycle}",
227 storageClass="Catalog",
228 dimensions=("instrument",),
229 )
230 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
231 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
232 name=f"fgcmStandardStars{cycle}",
233 storageClass="SimpleCatalog",
234 dimensions=("instrument",),
235 )
237 def __init__(self, *, config=None):
238 super().__init__(config=config)
240 if not config.doReferenceCalibration:
241 self.inputs.remove("fgcmReferenceStars")
242 self.inputs.remove("fgcmReferenceStarsParquet")
244 if config.useParquetCatalogFormat:
245 self.inputs.remove("fgcmStarObservations")
246 self.inputs.remove("fgcmStarIds")
247 self.inputs.remove("fgcmStarIndices")
248 if config.doReferenceCalibration:
249 self.inputs.remove("fgcmReferenceStars")
250 else:
251 self.inputs.remove("fgcmStarObservationsParquet")
252 self.inputs.remove("fgcmStarIdsParquet")
253 if config.doReferenceCalibration:
254 self.inputs.remove("fgcmReferenceStarsParquet")
256 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
257 raise ValueError("cycleNumber must be of integer format")
258 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
259 raise ValueError("previousCycleNumber must be of integer format")
260 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
261 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
263 if int(config.connections.cycleNumber) == 0:
264 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
265 self.prerequisiteInputs.remove("fgcmFitParametersInput")
267 if not self.config.doMultipleCycles:
268 # Single-cycle run
269 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
270 self.outputs.remove("fgcmStandardStars")
272 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
273 self.outputs.remove("fgcmZeropoints")
274 self.outputs.remove("fgcmAtmosphereParameters")
276 # Remove all multiple cycle outputs
277 for cycle in range(0, MULTIPLE_CYCLES_MAX):
278 self.outputs.remove(f"fgcmFitParameters{cycle}")
279 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
280 self.outputs.remove(f"fgcmZeropoints{cycle}")
281 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
282 self.outputs.remove(f"fgcmStandardStars{cycle}")
284 else:
285 # Multiple-cycle run
286 # Remove single-cycle outputs
287 self.outputs.remove("fgcmFitParameters")
288 self.outputs.remove("fgcmFlaggedStars")
289 self.outputs.remove("fgcmZeropoints")
290 self.outputs.remove("fgcmAtmosphereParameters")
291 self.outputs.remove("fgcmStandardStars")
293 # Remove outputs from cycles that are not used
294 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
295 MULTIPLE_CYCLES_MAX):
296 self.outputs.remove(f"fgcmFitParameters{cycle}")
297 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
298 self.outputs.remove(f"fgcmZeropoints{cycle}")
299 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
300 self.outputs.remove(f"fgcmStandardStars{cycle}")
302 # Remove non-final-cycle outputs if necessary
303 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
304 if not self.config.outputZeropointsBeforeFinalCycle:
305 self.outputs.remove(f"fgcmZeropoints{cycle}")
306 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
307 if not self.config.outputStandardsBeforeFinalCycle:
308 self.outputs.remove(f"fgcmStandardStars{cycle}")
311class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
312 pipelineConnections=FgcmFitCycleConnections):
313 """Config for FgcmFitCycle"""
315 doMultipleCycles = pexConfig.Field(
316 doc="Run multiple fit cycles in one task",
317 dtype=bool,
318 default=False,
319 )
320 useParquetCatalogFormat = pexConfig.Field(
321 doc="Use parquet catalog format?",
322 dtype=bool,
323 default=True,
324 )
325 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
326 doc=("Final cycle number in multiple cycle mode. The initial cycle "
327 "is 0, with limited parameters fit. The next cycle is 1 with "
328 "full parameter fit. The final cycle is a clean-up with no "
329 "parameters fit. There will be a total of "
330 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
331 "cycle number cannot be less than 2."),
332 dtype=int,
333 default=5,
334 min=2,
335 max=MULTIPLE_CYCLES_MAX,
336 inclusiveMax=True,
337 )
338 bands = pexConfig.ListField(
339 doc="Bands to run calibration",
340 dtype=str,
341 default=[],
342 )
343 fitBands = pexConfig.ListField(
344 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
345 "the atmosphere constrained from the 'fitBands' on the same night. "
346 "Must be a subset of `config.bands`"),
347 dtype=str,
348 default=[],
349 )
350 requiredBands = pexConfig.ListField(
351 doc=("Bands that are required for a star to be considered a calibration star. "
352 "Must be a subset of `config.bands`"),
353 dtype=str,
354 default=[],
355 )
356 physicalFilterMap = pexConfig.DictField(
357 doc="Mapping from 'physicalFilter' to band.",
358 keytype=str,
359 itemtype=str,
360 default={},
361 )
362 doReferenceCalibration = pexConfig.Field(
363 doc="Use reference catalog as additional constraint on calibration",
364 dtype=bool,
365 default=True,
366 )
367 refStarSnMin = pexConfig.Field(
368 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
369 dtype=float,
370 default=50.0,
371 )
372 refStarOutlierNSig = pexConfig.Field(
373 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
374 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
375 dtype=float,
376 default=4.0,
377 )
378 applyRefStarColorCuts = pexConfig.Field(
379 doc=("Apply color cuts defined in ``starColorCuts`` to reference stars? "
380 "These cuts are in addition to any cuts defined in ``refStarColorCuts``"),
381 dtype=bool,
382 default=True,
383 )
384 useExposureReferenceOffset = pexConfig.Field(
385 doc=("Use per-exposure (visit) offsets between calibrated stars and reference stars "
386 "for final zeropoints? This may help uniformity for disjoint surveys."),
387 dtype=bool,
388 default=False,
389 )
390 nCore = pexConfig.Field(
391 doc="Number of cores to use",
392 dtype=int,
393 default=4,
394 )
395 nStarPerRun = pexConfig.Field(
396 doc="Number of stars to run in each chunk",
397 dtype=int,
398 default=200000,
399 )
400 nExpPerRun = pexConfig.Field(
401 doc="Number of exposures to run in each chunk",
402 dtype=int,
403 default=1000,
404 )
405 reserveFraction = pexConfig.Field(
406 doc="Fraction of stars to reserve for testing",
407 dtype=float,
408 default=0.1,
409 )
410 freezeStdAtmosphere = pexConfig.Field(
411 doc="Freeze atmosphere parameters to standard (for testing)",
412 dtype=bool,
413 default=False,
414 )
415 precomputeSuperStarInitialCycle = pexConfig.Field(
416 doc="Precompute superstar flat for initial cycle",
417 dtype=bool,
418 default=False,
419 )
420 superStarSubCcdDict = pexConfig.DictField(
421 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
422 "Must have one entry per band."),
423 keytype=str,
424 itemtype=bool,
425 default={},
426 )
427 superStarSubCcdChebyshevOrder = pexConfig.Field(
428 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
429 "Global default is first-order polynomials, and should be overridden "
430 "on a camera-by-camera basis depending on the ISR."),
431 dtype=int,
432 default=1,
433 )
434 superStarSubCcdTriangular = pexConfig.Field(
435 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
436 "suppress high-order cross terms?"),
437 dtype=bool,
438 default=False,
439 )
440 superStarSigmaClip = pexConfig.Field(
441 doc="Number of sigma to clip outliers when selecting for superstar flats",
442 dtype=float,
443 default=5.0,
444 )
445 superStarPlotCcdResiduals = pexConfig.Field(
446 doc="If plotting is enabled, should per-detector residuals be plotted? "
447 "This may produce a lot of output, and should be used only for "
448 "debugging purposes.",
449 dtype=bool,
450 default=False,
451 )
452 focalPlaneSigmaClip = pexConfig.Field(
453 doc="Number of sigma to clip outliers per focal-plane.",
454 dtype=float,
455 default=4.0,
456 )
457 ccdGraySubCcdDict = pexConfig.DictField(
458 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
459 "('ccd gray') on a sub-ccd scale."),
460 keytype=str,
461 itemtype=bool,
462 default={},
463 )
464 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
465 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
466 dtype=int,
467 default=1,
468 )
469 ccdGraySubCcdTriangular = pexConfig.Field(
470 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
471 "suppress high-order cross terms?"),
472 dtype=bool,
473 default=True,
474 )
475 ccdGrayFocalPlaneDict = pexConfig.DictField(
476 doc=("Per-band specification on whether to compute focal-plane residual "
477 "('ccd gray') corrections."),
478 keytype=str,
479 itemtype=bool,
480 default={},
481 )
482 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
483 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
484 "gray corrections. If there are fewer good CCDs then the gray "
485 "correction is computed per-ccd."),
486 dtype=int,
487 default=1,
488 )
489 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
490 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
491 dtype=int,
492 default=3,
493 )
494 cycleNumber = pexConfig.Field(
495 doc=("FGCM fit cycle number. This is automatically incremented after each run "
496 "and stage of outlier rejection. See cookbook for details."),
497 dtype=int,
498 default=None,
499 )
500 isFinalCycle = pexConfig.Field(
501 doc=("Is this the final cycle of the fitting? Will automatically compute final "
502 "selection of stars and photometric exposures, and will output zeropoints "
503 "and standard stars for use in fgcmOutputProducts"),
504 dtype=bool,
505 default=False,
506 )
507 maxIterBeforeFinalCycle = pexConfig.Field(
508 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
509 "will always be 0 in the final cycle for cleanup and final selection."),
510 dtype=int,
511 default=50,
512 )
513 deltaMagBkgOffsetPercentile = pexConfig.Field(
514 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
515 "offset from local background subtraction."),
516 dtype=float,
517 default=0.25,
518 )
519 deltaMagBkgPerCcd = pexConfig.Field(
520 doc=("Compute net offset from local background subtraction per-ccd? "
521 "Otherwise, use computation per visit."),
522 dtype=bool,
523 default=False,
524 )
525 utBoundary = pexConfig.Field(
526 doc="Boundary (in UTC) from day-to-day",
527 dtype=float,
528 default=None,
529 )
530 washMjds = pexConfig.ListField(
531 doc="Mirror wash MJDs",
532 dtype=float,
533 default=(0.0,),
534 )
535 epochMjds = pexConfig.ListField(
536 doc="Epoch boundaries in MJD",
537 dtype=float,
538 default=(0.0,),
539 )
540 minObsPerBand = pexConfig.Field(
541 doc="Minimum good observations per band",
542 dtype=int,
543 default=2,
544 )
545 # TODO: When DM-16511 is done, it will be possible to get the
546 # telescope latitude directly from the camera.
547 latitude = pexConfig.Field(
548 doc="Observatory latitude",
549 dtype=float,
550 default=None,
551 )
552 mirrorArea = pexConfig.Field(
553 doc="Mirror area (square meters) of telescope. If not set, will "
554 "try to estimate from camera.telescopeDiameter.",
555 dtype=float,
556 default=None,
557 optional=True,
558 )
559 defaultCameraOrientation = pexConfig.Field(
560 doc="Default camera orientation for QA plots.",
561 dtype=float,
562 default=None,
563 )
564 brightObsGrayMax = pexConfig.Field(
565 doc="Maximum gray extinction to be considered bright observation",
566 dtype=float,
567 default=0.15,
568 )
569 minStarPerCcd = pexConfig.Field(
570 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
571 "CCDs with fewer stars will have their calibration estimated from other "
572 "CCDs in the same visit, with zeropoint error increased accordingly."),
573 dtype=int,
574 default=5,
575 )
576 minCcdPerExp = pexConfig.Field(
577 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
578 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
579 dtype=int,
580 default=5,
581 )
582 maxCcdGrayErr = pexConfig.Field(
583 doc="Maximum error on CCD gray offset to be considered photometric",
584 dtype=float,
585 default=0.05,
586 )
587 minStarPerExp = pexConfig.Field(
588 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
589 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
590 dtype=int,
591 default=600,
592 )
593 minExpPerNight = pexConfig.Field(
594 doc="Minimum number of good exposures/visits to consider a partly photometric night",
595 dtype=int,
596 default=10,
597 )
598 expGrayInitialCut = pexConfig.Field(
599 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
600 "observations."),
601 dtype=float,
602 default=-0.25,
603 )
604 expGrayPhotometricCutDict = pexConfig.DictField(
605 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
606 "('gray term') for a visit to be considered photometric. Must have one "
607 "entry per band. Broad-band filters should be -0.05."),
608 keytype=str,
609 itemtype=float,
610 default={},
611 )
612 expGrayHighCutDict = pexConfig.DictField(
613 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
614 "('gray term') for a visit to be considered photometric. Must have one "
615 "entry per band. Broad-band filters should be 0.2."),
616 keytype=str,
617 itemtype=float,
618 default={},
619 )
620 expGrayRecoverCut = pexConfig.Field(
621 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
622 "Visits with more gray extinction will only get CCD zeropoints if there are "
623 "sufficient star observations (minStarPerCcd) on that CCD."),
624 dtype=float,
625 default=-1.0,
626 )
627 expVarGrayPhotometricCutDict = pexConfig.DictField(
628 doc=("Per-band specification on maximum exposure variance to be considered possibly "
629 "photometric. Must have one entry per band. Broad-band filters should be "
630 "0.0005."),
631 keytype=str,
632 itemtype=float,
633 default={},
634 )
635 expGrayErrRecoverCut = pexConfig.Field(
636 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
637 "Visits with more gray variance will only get CCD zeropoints if there are "
638 "sufficient star observations (minStarPerCcd) on that CCD."),
639 dtype=float,
640 default=0.05,
641 )
642 aperCorrFitNBins = pexConfig.Field(
643 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
644 "no fit will be performed, and the config.aperCorrInputSlopes will be "
645 "used if available."),
646 dtype=int,
647 default=10,
648 )
649 aperCorrInputSlopeDict = pexConfig.DictField(
650 doc=("Per-band specification of aperture correction input slope parameters. These "
651 "are used on the first fit iteration, and aperture correction parameters will "
652 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
653 "to set this when there is insufficient data to fit the parameters (e.g. "
654 "tract mode)."),
655 keytype=str,
656 itemtype=float,
657 default={},
658 )
659 sedboundaryterms = pexConfig.ConfigField(
660 doc="Mapping from bands to SED boundary term names used is sedterms.",
661 dtype=SedboundarytermDict,
662 )
663 sedterms = pexConfig.ConfigField(
664 doc="Mapping from terms to bands for fgcm linear SED approximations.",
665 dtype=SedtermDict,
666 )
667 sigFgcmMaxErr = pexConfig.Field(
668 doc="Maximum mag error for fitting sigma_FGCM",
669 dtype=float,
670 default=0.01,
671 )
672 sigFgcmMaxEGrayDict = pexConfig.DictField(
673 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
674 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
675 "should be 0.05."),
676 keytype=str,
677 itemtype=float,
678 default={},
679 )
680 ccdGrayMaxStarErr = pexConfig.Field(
681 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
682 "computation"),
683 dtype=float,
684 default=0.10,
685 )
686 approxThroughputDict = pexConfig.DictField(
687 doc=("Per-band specification of the approximate overall throughput at the start of "
688 "calibration observations. Must have one entry per band. Typically should "
689 "be 1.0."),
690 keytype=str,
691 itemtype=float,
692 default={},
693 )
694 sigmaCalRange = pexConfig.ListField(
695 doc="Allowed range for systematic error floor estimation",
696 dtype=float,
697 default=(0.001, 0.003),
698 )
699 sigmaCalFitPercentile = pexConfig.ListField(
700 doc="Magnitude percentile range to fit systematic error floor",
701 dtype=float,
702 default=(0.05, 0.15),
703 )
704 sigmaCalPlotPercentile = pexConfig.ListField(
705 doc="Magnitude percentile range to plot systematic error floor",
706 dtype=float,
707 default=(0.05, 0.95),
708 )
709 sigma0Phot = pexConfig.Field(
710 doc="Systematic error floor for all zeropoints",
711 dtype=float,
712 default=0.003,
713 )
714 mapLongitudeRef = pexConfig.Field(
715 doc="Reference longitude for plotting maps",
716 dtype=float,
717 default=0.0,
718 )
719 mapNSide = pexConfig.Field(
720 doc="Healpix nside for plotting maps",
721 dtype=int,
722 default=256,
723 )
724 outfileBase = pexConfig.Field(
725 doc="Filename start for plot output files",
726 dtype=str,
727 default=None,
728 )
729 starColorCuts = pexConfig.ListField(
730 doc=("Encoded star-color cuts (using calibration star colors). "
731 "This is a list with each entry a string of the format "
732 "``band1,band2,low,high`` such that only stars of color "
733 "low < band1 - band2 < high will be used for calibration."),
734 dtype=str,
735 default=("NO_DATA",),
736 )
737 refStarColorCuts = pexConfig.ListField(
738 doc=("Encoded star color cuts specifically to apply to reference stars. "
739 "This is a list with each entry a string of the format "
740 "``band1,band2,low,high`` such that only stars of color "
741 "low < band1 - band2 < high will be used as reference stars."),
742 dtype=str,
743 default=("NO_DATA",),
744 )
745 colorSplitBands = pexConfig.ListField(
746 doc="Band names to use to split stars by color. Must have 2 entries.",
747 dtype=str,
748 length=2,
749 default=('g', 'i'),
750 )
751 modelMagErrors = pexConfig.Field(
752 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
753 dtype=bool,
754 default=True,
755 )
756 useQuadraticPwv = pexConfig.Field(
757 doc="Model PWV with a quadratic term for variation through the night?",
758 dtype=bool,
759 default=False,
760 )
761 instrumentParsPerBand = pexConfig.Field(
762 doc=("Model instrumental parameters per band? "
763 "Otherwise, instrumental parameters (QE changes with time) are "
764 "shared among all bands."),
765 dtype=bool,
766 default=False,
767 )
768 instrumentSlopeMinDeltaT = pexConfig.Field(
769 doc=("Minimum time change (in days) between observations to use in constraining "
770 "instrument slope."),
771 dtype=float,
772 default=20.0,
773 )
774 fitMirrorChromaticity = pexConfig.Field(
775 doc="Fit (intraband) mirror chromatic term?",
776 dtype=bool,
777 default=False,
778 )
779 coatingMjds = pexConfig.ListField(
780 doc="Mirror coating dates in MJD",
781 dtype=float,
782 default=(0.0,),
783 )
784 outputStandardsBeforeFinalCycle = pexConfig.Field(
785 doc="Output standard stars prior to final cycle? Used in debugging.",
786 dtype=bool,
787 default=False,
788 )
789 outputZeropointsBeforeFinalCycle = pexConfig.Field(
790 doc="Output standard stars prior to final cycle? Used in debugging.",
791 dtype=bool,
792 default=False,
793 )
794 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
795 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
796 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
797 keytype=str,
798 itemtype=bool,
799 default={},
800 )
801 autoPhotometricCutNSig = pexConfig.Field(
802 doc=("Number of sigma for automatic computation of (low) photometric cut. "
803 "Cut is based on exposure gray width (per band), unless "
804 "useRepeatabilityForExpGrayCuts is set, in which case the star "
805 "repeatability is used (also per band)."),
806 dtype=float,
807 default=3.0,
808 )
809 autoHighCutNSig = pexConfig.Field(
810 doc=("Number of sigma for automatic computation of (high) outlier cut. "
811 "Cut is based on exposure gray width (per band), unless "
812 "useRepeatabilityForExpGrayCuts is set, in which case the star "
813 "repeatability is used (also per band)."),
814 dtype=float,
815 default=4.0,
816 )
817 quietMode = pexConfig.Field(
818 doc="Be less verbose with logging.",
819 dtype=bool,
820 default=False,
821 )
822 doPlots = pexConfig.Field(
823 doc="Make fgcm QA plots.",
824 dtype=bool,
825 default=True,
826 )
827 randomSeed = pexConfig.Field(
828 doc="Random seed for fgcm for consistency in tests.",
829 dtype=int,
830 default=None,
831 optional=True,
832 )
833 deltaAperFitMinNgoodObs = pexConfig.Field(
834 doc="Minimum number of good observations to use mean delta-aper values in fits.",
835 dtype=int,
836 default=2,
837 )
838 deltaAperFitPerCcdNx = pexConfig.Field(
839 doc=("Number of x bins per ccd when computing delta-aper background offsets. "
840 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
841 dtype=int,
842 default=10,
843 )
844 deltaAperFitPerCcdNy = pexConfig.Field(
845 doc=("Number of y bins per ccd when computing delta-aper background offsets. "
846 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
847 dtype=int,
848 default=10,
849 )
850 deltaAperFitSpatialNside = pexConfig.Field(
851 doc="Healpix nside to compute spatial delta-aper background offset maps.",
852 dtype=int,
853 default=64,
854 )
855 deltaAperInnerRadiusArcsec = pexConfig.Field(
856 doc=("Inner radius used to compute deltaMagAper (arcseconds). "
857 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if "
858 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
859 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
860 dtype=float,
861 default=0.0,
862 )
863 deltaAperOuterRadiusArcsec = pexConfig.Field(
864 doc=("Outer radius used to compute deltaMagAper (arcseconds). "
865 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if "
866 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
867 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
868 dtype=float,
869 default=0.0,
870 )
871 doComputeDeltaAperPerVisit = pexConfig.Field(
872 doc=("Do the computation of delta-aper background offsets per visit? "
873 "Note: this option can be very slow when there are many visits."),
874 dtype=bool,
875 default=False,
876 )
877 doComputeDeltaAperPerStar = pexConfig.Field(
878 doc="Do the computation of delta-aper mean values per star?",
879 dtype=bool,
880 default=True,
881 )
882 doComputeDeltaAperMap = pexConfig.Field(
883 doc=("Do the computation of delta-aper spatial maps? "
884 "This is only used if ``doComputeDeltaAperPerStar`` is True,"),
885 dtype=bool,
886 default=False,
887 )
888 doComputeDeltaAperPerCcd = pexConfig.Field(
889 doc="Do the computation of per-ccd delta-aper background offsets?",
890 dtype=bool,
891 default=False,
892 )
894 def validate(self):
895 super().validate()
897 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
898 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
899 raise RuntimeError(msg)
900 if self.connections.cycleNumber != str(self.cycleNumber):
901 msg = "cycleNumber in template must be equal to connections.cycleNumber"
902 raise RuntimeError(msg)
904 for band in self.fitBands:
905 if band not in self.bands:
906 msg = 'fitBand %s not in bands' % (band)
907 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
908 for band in self.requiredBands:
909 if band not in self.bands:
910 msg = 'requiredBand %s not in bands' % (band)
911 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
912 for band in self.colorSplitBands:
913 if band not in self.bands:
914 msg = 'colorSplitBand %s not in bands' % (band)
915 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
916 for band in self.bands:
917 if band not in self.superStarSubCcdDict:
918 msg = 'band %s not in superStarSubCcdDict' % (band)
919 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
920 self, msg)
921 if band not in self.ccdGraySubCcdDict:
922 msg = 'band %s not in ccdGraySubCcdDict' % (band)
923 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
924 self, msg)
925 if band not in self.expGrayPhotometricCutDict:
926 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
927 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
928 self, msg)
929 if band not in self.expGrayHighCutDict:
930 msg = 'band %s not in expGrayHighCutDict' % (band)
931 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
932 self, msg)
933 if band not in self.expVarGrayPhotometricCutDict:
934 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
935 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
936 self, msg)
937 if band not in self.sigFgcmMaxEGrayDict:
938 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
939 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
940 self, msg)
941 if band not in self.approxThroughputDict:
942 msg = 'band %s not in approxThroughputDict' % (band)
943 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
944 self, msg)
945 if band not in self.useRepeatabilityForExpGrayCutsDict:
946 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
947 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
948 self, msg)
950 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \
951 or self.doComputeDeltaAperPerCcd:
952 if self.deltaAperInnerRadiusArcsec <= 0.0:
953 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.'
954 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec,
955 self, msg)
956 if self.deltaAperOuterRadiusArcsec <= 0.0:
957 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.'
958 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
959 self, msg)
960 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec:
961 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if '
962 'deltaAper computations are turned on.')
963 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
964 self, msg)
967class FgcmFitCycleTask(pipeBase.PipelineTask):
968 """
969 Run Single fit cycle for FGCM global calibration
970 """
972 ConfigClass = FgcmFitCycleConfig
973 _DefaultName = "fgcmFitCycle"
975 def __init__(self, initInputs=None, **kwargs):
976 super().__init__(**kwargs)
978 def runQuantum(self, butlerQC, inputRefs, outputRefs):
979 camera = butlerQC.get(inputRefs.camera)
981 handleDict = {}
983 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
984 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
986 if self.config.useParquetCatalogFormat:
987 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservationsParquet)
988 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIdsParquet)
989 if self.config.doReferenceCalibration:
990 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStarsParquet)
991 else:
992 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
993 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
994 handleDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
995 if self.config.doReferenceCalibration:
996 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
997 if self.config.cycleNumber > 0:
998 handleDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
999 handleDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
1001 fgcmDatasetDict = None
1002 if self.config.doMultipleCycles:
1003 # Run multiple cycles at once.
1004 config = copy.copy(self.config)
1005 config.update(cycleNumber=0)
1006 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
1007 if cycle == self.config.multipleCyclesFinalCycleNumber:
1008 config.update(isFinalCycle=True)
1010 if cycle > 0:
1011 handleDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
1012 handleDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
1014 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config)
1015 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
1016 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
1017 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
1018 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
1019 if self.outputZeropoints:
1020 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
1021 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
1022 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
1023 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
1024 if self.outputStandards:
1025 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
1026 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
1027 else:
1028 # Run a single cycle
1029 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict)
1031 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
1032 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
1033 if self.outputZeropoints:
1034 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
1035 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
1036 if self.outputStandards:
1037 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
1039 def _fgcmFitCycle(self, camera, handleDict, config=None):
1040 """
1041 Run the fit cycle
1043 Parameters
1044 ----------
1045 camera : `lsst.afw.cameraGeom.Camera`
1046 handleDict : `dict`
1047 All handles are `lsst.daf.butler.DeferredDatasetHandle`
1048 handle dictionary with keys:
1050 ``"fgcmLookUpTable"``
1051 handle for the FGCM look-up table.
1052 ``"fgcmVisitCatalog"``
1053 handle for visit summary catalog.
1054 ``"fgcmStarObservations"``
1055 handle for star observation catalog.
1056 ``"fgcmStarIds"``
1057 handle for star id catalog.
1058 ``"fgcmStarIndices"``
1059 handle for star index catalog.
1060 ``"fgcmReferenceStars"``
1061 handle for matched reference star catalog.
1062 ``"fgcmFlaggedStars"``
1063 handle for flagged star catalog.
1064 ``"fgcmFitParameters"``
1065 handle for fit parameter catalog.
1066 config : `lsst.pex.config.Config`, optional
1067 Configuration to use to override self.config.
1069 Returns
1070 -------
1071 fgcmDatasetDict : `dict`
1072 Dictionary of datasets to persist.
1073 """
1074 if config is not None:
1075 _config = config
1076 else:
1077 _config = self.config
1079 # Set defaults on whether to output standards and zeropoints
1080 self.maxIter = _config.maxIterBeforeFinalCycle
1081 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1082 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1083 self.resetFitParameters = True
1085 if _config.isFinalCycle:
1086 # This is the final fit cycle, so we do not want to reset fit
1087 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1088 # and we always want to output standards and zeropoints
1089 self.maxIter = 0
1090 self.outputStandards = True
1091 self.outputZeropoints = True
1092 self.resetFitParameters = False
1094 lutCat = handleDict['fgcmLookUpTable'].get()
1095 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1096 dict(_config.physicalFilterMap))
1097 del lutCat
1099 configDict = makeConfigDict(_config, self.log, camera,
1100 self.maxIter, self.resetFitParameters,
1101 self.outputZeropoints,
1102 lutIndexVals[0]['FILTERNAMES'])
1104 # next we need the exposure/visit information
1105 visitCat = handleDict['fgcmVisitCatalog'].get()
1106 fgcmExpInfo = translateVisitCatalog(visitCat)
1107 del visitCat
1109 focalPlaneProjector = FocalPlaneProjector(camera,
1110 self.config.defaultCameraOrientation)
1112 noFitsDict = {'lutIndex': lutIndexVals,
1113 'lutStd': lutStd,
1114 'expInfo': fgcmExpInfo,
1115 'focalPlaneProjector': focalPlaneProjector}
1117 # set up the fitter object
1118 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1119 noFitsDict=noFitsDict, noOutput=True)
1121 # create the parameter object
1122 if (fgcmFitCycle.initialCycle):
1123 # cycle = 0, initial cycle
1124 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1125 fgcmLut,
1126 fgcmExpInfo)
1127 else:
1128 if isinstance(handleDict['fgcmFitParameters'], afwTable.BaseCatalog):
1129 parCat = handleDict['fgcmFitParameters']
1130 else:
1131 parCat = handleDict['fgcmFitParameters'].get()
1132 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1133 del parCat
1134 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1135 fgcmExpInfo,
1136 inParInfo,
1137 inParams,
1138 inSuperStar)
1140 # set up the stars...
1141 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1143 starObs = handleDict['fgcmStarObservations'].get()
1144 starIds = handleDict['fgcmStarIds'].get()
1145 if not self.config.useParquetCatalogFormat:
1146 starIndices = handleDict['fgcmStarIndices'].get()
1147 else:
1148 starIndices = None
1150 # grab the flagged stars if available
1151 if 'fgcmFlaggedStars' in handleDict:
1152 if isinstance(handleDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1153 flaggedStars = handleDict['fgcmFlaggedStars']
1154 else:
1155 flaggedStars = handleDict['fgcmFlaggedStars'].get()
1156 flagId = flaggedStars['objId'][:]
1157 flagFlag = flaggedStars['objFlag'][:]
1159 del flaggedStars
1160 elif self.config.useParquetCatalogFormat:
1161 # If we are using the parquet catalog format, then that means that
1162 # reserved stars have already been flagged. We extract the flags here
1163 # to input to fgcm, which will then be persisted (with additional
1164 # quality flags) as the fgcmFlaggedStars datatype in subsequent
1165 # fit cycles.
1166 (flagged,) = (starIds['obj_flag'] > 0).nonzero()
1167 flagId = starIds['fgcm_id'][flagged]
1168 flagFlag = starIds['obj_flag'][flagged]
1169 else:
1170 flagId = None
1171 flagFlag = None
1173 if _config.doReferenceCalibration:
1174 refStars = handleDict['fgcmReferenceStars'].get()
1176 refMag, refMagErr = extractReferenceMags(refStars,
1177 _config.bands,
1178 _config.physicalFilterMap)
1180 refId = refStars['fgcm_id'][:]
1181 else:
1182 refStars = None
1183 refId = None
1184 refMag = None
1185 refMagErr = None
1187 # match star observations to visits
1188 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1189 # actually be transferred into fgcm using the indexing below.
1190 if self.config.useParquetCatalogFormat:
1191 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'])
1192 else:
1193 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1195 # The fgcmStars.loadStars method will copy all the star information into
1196 # special shared memory objects that will not blow up the memory usage when
1197 # used with python multiprocessing. Once all the numbers are copied,
1198 # it is necessary to release all references to the objects that previously
1199 # stored the data to ensure that the garbage collector can clear the memory,
1200 # and ensure that this memory is not copied when multiprocessing kicks in.
1202 if self.config.useParquetCatalogFormat:
1203 # Note that the ra/dec coordinates for the parquet format are in
1204 # degrees, which is what fgcm expects.
1205 fgcmStars.loadStars(fgcmPars,
1206 starObs['visit'][:],
1207 starObs['detector'][:],
1208 starObs['ra'][:],
1209 starObs['dec'][:],
1210 starObs['inst_mag'][:],
1211 starObs['inst_mag_err'][:],
1212 fgcmExpInfo['FILTERNAME'][visitIndex],
1213 starIds['fgcm_id'][:],
1214 starIds['ra'][:],
1215 starIds['dec'][:],
1216 starIds['obs_arr_index'][:],
1217 starIds['n_obs'][:],
1218 obsX=starObs['x'][:],
1219 obsY=starObs['y'][:],
1220 obsDeltaMagBkg=starObs['delta_mag_bkg'][:],
1221 obsDeltaAper=starObs['delta_mag_aper'][:],
1222 refID=refId,
1223 refMag=refMag,
1224 refMagErr=refMagErr,
1225 flagID=flagId,
1226 flagFlag=flagFlag,
1227 computeNobs=True)
1228 else:
1229 # We determine the conversion from the native units (typically radians) to
1230 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1231 # numpy arrays rather than Angles, which would we approximately 600x slower.
1232 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1234 fgcmStars.loadStars(fgcmPars,
1235 starObs['visit'][starIndices['obsIndex']],
1236 starObs['ccd'][starIndices['obsIndex']],
1237 starObs['ra'][starIndices['obsIndex']] * conv,
1238 starObs['dec'][starIndices['obsIndex']] * conv,
1239 starObs['instMag'][starIndices['obsIndex']],
1240 starObs['instMagErr'][starIndices['obsIndex']],
1241 fgcmExpInfo['FILTERNAME'][visitIndex],
1242 starIds['fgcm_id'][:],
1243 starIds['ra'][:],
1244 starIds['dec'][:],
1245 starIds['obsArrIndex'][:],
1246 starIds['nObs'][:],
1247 obsX=starObs['x'][starIndices['obsIndex']],
1248 obsY=starObs['y'][starIndices['obsIndex']],
1249 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1250 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']],
1251 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1252 refID=refId,
1253 refMag=refMag,
1254 refMagErr=refMagErr,
1255 flagID=flagId,
1256 flagFlag=flagFlag,
1257 computeNobs=True)
1259 # Release all references to temporary objects holding star data (see above)
1260 del starObs
1261 del starIds
1262 del starIndices
1263 del flagId
1264 del flagFlag
1265 del refStars
1266 del refId
1267 del refMag
1268 del refMagErr
1270 # and set the bits in the cycle object
1271 fgcmFitCycle.setLUT(fgcmLut)
1272 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1273 fgcmFitCycle.setPars(fgcmPars)
1275 # finish the setup
1276 fgcmFitCycle.finishSetup()
1278 # and run
1279 fgcmFitCycle.run()
1281 ##################
1282 # Persistance
1283 ##################
1285 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1287 # Output the config for the next cycle
1288 # We need to make a copy since the input one has been frozen
1290 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1291 i, b in enumerate(_config.bands)}
1292 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1293 i, band in enumerate(_config.bands)}
1295 outConfig = copy.copy(_config)
1296 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1297 precomputeSuperStarInitialCycle=False,
1298 freezeStdAtmosphere=False,
1299 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1300 expGrayHighCutDict=updatedHighCutDict)
1302 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1303 cycleNumber=str(_config.cycleNumber + 1))
1305 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1306 outConfig.cycleNumber)
1307 outConfig.save(configFileName)
1309 if _config.isFinalCycle == 1:
1310 # We are done, ready to output products
1311 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1312 else:
1313 self.log.info("Saved config for next cycle to %s" % (configFileName))
1314 self.log.info("Be sure to look at:")
1315 self.log.info(" config.expGrayPhotometricCut")
1316 self.log.info(" config.expGrayHighCut")
1317 self.log.info("If you are satisfied with the fit, please set:")
1318 self.log.info(" config.isFinalCycle = True")
1320 fgcmFitCycle.freeSharedMemory()
1322 return fgcmDatasetDict, outConfig
1324 def _loadParameters(self, parCat):
1325 """
1326 Load FGCM parameters from a previous fit cycle
1328 Parameters
1329 ----------
1330 parCat : `lsst.afw.table.BaseCatalog`
1331 Parameter catalog in afw table form.
1333 Returns
1334 -------
1335 inParInfo: `numpy.ndarray`
1336 Numpy array parameter information formatted for input to fgcm
1337 inParameters: `numpy.ndarray`
1338 Numpy array parameter values formatted for input to fgcm
1339 inSuperStar: `numpy.array`
1340 Superstar flat formatted for input to fgcm
1341 """
1342 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1343 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1345 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1346 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1347 (parLutFilterNames.size, )),
1348 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1349 ('LNTAUUNIT', 'f8'),
1350 ('LNTAUSLOPEUNIT', 'f8'),
1351 ('ALPHAUNIT', 'f8'),
1352 ('LNPWVUNIT', 'f8'),
1353 ('LNPWVSLOPEUNIT', 'f8'),
1354 ('LNPWVQUADRATICUNIT', 'f8'),
1355 ('LNPWVGLOBALUNIT', 'f8'),
1356 ('O3UNIT', 'f8'),
1357 ('QESYSUNIT', 'f8'),
1358 ('FILTEROFFSETUNIT', 'f8'),
1359 ('HASEXTERNALPWV', 'i2'),
1360 ('HASEXTERNALTAU', 'i2')])
1361 inParInfo['NCCD'] = parCat['nCcd']
1362 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1363 inParInfo['FITBANDS'][:] = parFitBands
1364 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1365 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1367 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1368 ('PARO3', 'f8', (parCat['parO3'].size, )),
1369 ('PARLNTAUINTERCEPT', 'f8',
1370 (parCat['parLnTauIntercept'].size, )),
1371 ('PARLNTAUSLOPE', 'f8',
1372 (parCat['parLnTauSlope'].size, )),
1373 ('PARLNPWVINTERCEPT', 'f8',
1374 (parCat['parLnPwvIntercept'].size, )),
1375 ('PARLNPWVSLOPE', 'f8',
1376 (parCat['parLnPwvSlope'].size, )),
1377 ('PARLNPWVQUADRATIC', 'f8',
1378 (parCat['parLnPwvQuadratic'].size, )),
1379 ('PARQESYSINTERCEPT', 'f8',
1380 (parCat['parQeSysIntercept'].size, )),
1381 ('COMPQESYSSLOPE', 'f8',
1382 (parCat['compQeSysSlope'].size, )),
1383 ('PARFILTEROFFSET', 'f8',
1384 (parCat['parFilterOffset'].size, )),
1385 ('PARFILTEROFFSETFITFLAG', 'i2',
1386 (parCat['parFilterOffsetFitFlag'].size, )),
1387 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1388 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1389 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1390 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1391 ('COMPABSTHROUGHPUT', 'f8',
1392 (parCat['compAbsThroughput'].size, )),
1393 ('COMPREFOFFSET', 'f8',
1394 (parCat['compRefOffset'].size, )),
1395 ('COMPREFSIGMA', 'f8',
1396 (parCat['compRefSigma'].size, )),
1397 ('COMPMIRRORCHROMATICITY', 'f8',
1398 (parCat['compMirrorChromaticity'].size, )),
1399 ('MIRRORCHROMATICITYPIVOT', 'f8',
1400 (parCat['mirrorChromaticityPivot'].size, )),
1401 ('COMPMEDIANSEDSLOPE', 'f8',
1402 (parCat['compMedianSedSlope'].size, )),
1403 ('COMPAPERCORRPIVOT', 'f8',
1404 (parCat['compAperCorrPivot'].size, )),
1405 ('COMPAPERCORRSLOPE', 'f8',
1406 (parCat['compAperCorrSlope'].size, )),
1407 ('COMPAPERCORRSLOPEERR', 'f8',
1408 (parCat['compAperCorrSlopeErr'].size, )),
1409 ('COMPAPERCORRRANGE', 'f8',
1410 (parCat['compAperCorrRange'].size, )),
1411 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1412 (parCat['compModelErrExptimePivot'].size, )),
1413 ('COMPMODELERRFWHMPIVOT', 'f8',
1414 (parCat['compModelErrFwhmPivot'].size, )),
1415 ('COMPMODELERRSKYPIVOT', 'f8',
1416 (parCat['compModelErrSkyPivot'].size, )),
1417 ('COMPMODELERRPARS', 'f8',
1418 (parCat['compModelErrPars'].size, )),
1419 ('COMPEXPGRAY', 'f8',
1420 (parCat['compExpGray'].size, )),
1421 ('COMPVARGRAY', 'f8',
1422 (parCat['compVarGray'].size, )),
1423 ('COMPEXPDELTAMAGBKG', 'f8',
1424 (parCat['compExpDeltaMagBkg'].size, )),
1425 ('COMPNGOODSTARPEREXP', 'i4',
1426 (parCat['compNGoodStarPerExp'].size, )),
1427 ('COMPEXPREFOFFSET', 'f8',
1428 (parCat['compExpRefOffset'].size, )),
1429 ('COMPSIGFGCM', 'f8',
1430 (parCat['compSigFgcm'].size, )),
1431 ('COMPSIGMACAL', 'f8',
1432 (parCat['compSigmaCal'].size, )),
1433 ('COMPRETRIEVEDLNPWV', 'f8',
1434 (parCat['compRetrievedLnPwv'].size, )),
1435 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1436 (parCat['compRetrievedLnPwvRaw'].size, )),
1437 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1438 (parCat['compRetrievedLnPwvFlag'].size, )),
1439 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1440 (parCat['compRetrievedTauNight'].size, )),
1441 ('COMPEPSILON', 'f8',
1442 (parCat['compEpsilon'].size, )),
1443 ('COMPMEDDELTAAPER', 'f8',
1444 (parCat['compMedDeltaAper'].size, )),
1445 ('COMPGLOBALEPSILON', 'f4',
1446 (parCat['compGlobalEpsilon'].size, )),
1447 ('COMPEPSILONMAP', 'f4',
1448 (parCat['compEpsilonMap'].size, )),
1449 ('COMPEPSILONNSTARMAP', 'i4',
1450 (parCat['compEpsilonNStarMap'].size, )),
1451 ('COMPEPSILONCCDMAP', 'f4',
1452 (parCat['compEpsilonCcdMap'].size, )),
1453 ('COMPEPSILONCCDNSTARMAP', 'i4',
1454 (parCat['compEpsilonCcdNStarMap'].size, ))])
1456 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1457 inParams['PARO3'][:] = parCat['parO3'][0, :]
1458 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1459 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1460 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1461 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1462 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1463 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1464 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1465 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1466 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1467 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1468 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1469 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1470 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1471 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1472 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1473 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1474 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1475 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1476 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1477 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1478 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1479 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1480 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1481 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1482 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1483 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1484 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1485 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1486 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1487 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1488 inParams['COMPEXPREFOFFSET'][:] = parCat['compExpRefOffset'][0, :]
1489 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1490 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1491 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1492 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1493 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1494 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1495 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :]
1496 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :]
1497 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :]
1498 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :]
1499 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :]
1500 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :]
1501 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :]
1503 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1504 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1506 return (inParInfo, inParams, inSuperStar)
1508 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1509 """
1510 Persist FGCM datasets through the butler.
1512 Parameters
1513 ----------
1514 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1515 Fgcm Fit cycle object
1516 """
1517 fgcmDatasetDict = {}
1519 # Save the parameters
1520 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1522 parSchema = afwTable.Schema()
1524 comma = ','
1525 lutFilterNameString = comma.join([n.decode('utf-8')
1526 for n in parInfo['LUTFILTERNAMES'][0]])
1527 fitBandString = comma.join([n.decode('utf-8')
1528 for n in parInfo['FITBANDS'][0]])
1530 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1531 lutFilterNameString, fitBandString)
1532 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1533 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1534 lutFilterNameString, fitBandString)
1536 fgcmDatasetDict['fgcmFitParameters'] = parCat
1538 # Save the indices of the flagged stars
1539 # (stars that have been (a) reserved from the fit for testing and
1540 # (b) bad stars that have failed quality checks.)
1541 flagStarSchema = self._makeFlagStarSchema()
1542 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1543 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1545 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1547 # Save the zeropoint information and atmospheres only if desired
1548 if self.outputZeropoints:
1549 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1550 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1552 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1553 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1555 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1557 # Save atmosphere values
1558 # These are generated by the same code that generates zeropoints
1559 atmSchema = makeAtmSchema()
1560 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1562 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1564 # Save the standard stars (if configured)
1565 if self.outputStandards:
1566 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1567 stdSchema = makeStdSchema(len(goodBands))
1568 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1570 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1572 return fgcmDatasetDict
1574 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1575 lutFilterNameString, fitBandString):
1576 """
1577 Make the parameter persistence schema
1579 Parameters
1580 ----------
1581 parInfo: `numpy.ndarray`
1582 Parameter information returned by fgcm
1583 pars: `numpy.ndarray`
1584 Parameter values returned by fgcm
1585 parSuperStarFlat: `numpy.array`
1586 Superstar flat values returned by fgcm
1587 lutFilterNameString: `str`
1588 Combined string of all the lutFilterNames
1589 fitBandString: `str`
1590 Combined string of all the fitBands
1592 Returns
1593 -------
1594 parSchema: `afwTable.schema`
1595 """
1597 parSchema = afwTable.Schema()
1599 # parameter info section
1600 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1601 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1602 size=len(lutFilterNameString))
1603 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1604 size=len(fitBandString))
1605 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1606 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1607 doc='Step units for ln(AOD) slope')
1608 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1609 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1610 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1611 doc='Step units for ln(pwv) slope')
1612 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1613 doc='Step units for ln(pwv) quadratic term')
1614 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1615 doc='Step units for global ln(pwv) parameters')
1616 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1617 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1618 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1619 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1620 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1622 # parameter section
1623 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1624 size=pars['PARALPHA'].size)
1625 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1626 size=pars['PARO3'].size)
1627 parSchema.addField('parLnTauIntercept', type='ArrayD',
1628 doc='ln(Tau) intercept parameter vector',
1629 size=pars['PARLNTAUINTERCEPT'].size)
1630 parSchema.addField('parLnTauSlope', type='ArrayD',
1631 doc='ln(Tau) slope parameter vector',
1632 size=pars['PARLNTAUSLOPE'].size)
1633 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1634 size=pars['PARLNPWVINTERCEPT'].size)
1635 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1636 size=pars['PARLNPWVSLOPE'].size)
1637 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1638 size=pars['PARLNPWVQUADRATIC'].size)
1639 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1640 size=pars['PARQESYSINTERCEPT'].size)
1641 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1642 size=pars[0]['COMPQESYSSLOPE'].size)
1643 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1644 size=pars['PARFILTEROFFSET'].size)
1645 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1646 size=pars['PARFILTEROFFSETFITFLAG'].size)
1647 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1648 doc='Global scale for retrieved ln(pwv)')
1649 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1650 doc='Global offset for retrieved ln(pwv)')
1651 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1652 doc='Nightly offset for retrieved ln(pwv)',
1653 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1654 parSchema.addField('compAbsThroughput', type='ArrayD',
1655 doc='Absolute throughput (relative to transmission curves)',
1656 size=pars['COMPABSTHROUGHPUT'].size)
1657 parSchema.addField('compRefOffset', type='ArrayD',
1658 doc='Offset between reference stars and calibrated stars',
1659 size=pars['COMPREFOFFSET'].size)
1660 parSchema.addField('compRefSigma', type='ArrayD',
1661 doc='Width of reference star/calibrated star distribution',
1662 size=pars['COMPREFSIGMA'].size)
1663 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1664 doc='Computed mirror chromaticity terms',
1665 size=pars['COMPMIRRORCHROMATICITY'].size)
1666 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1667 doc='Mirror chromaticity pivot mjd',
1668 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1669 parSchema.addField('compMedianSedSlope', type='ArrayD',
1670 doc='Computed median SED slope (per band)',
1671 size=pars['COMPMEDIANSEDSLOPE'].size)
1672 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1673 size=pars['COMPAPERCORRPIVOT'].size)
1674 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1675 size=pars['COMPAPERCORRSLOPE'].size)
1676 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1677 size=pars['COMPAPERCORRSLOPEERR'].size)
1678 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1679 size=pars['COMPAPERCORRRANGE'].size)
1680 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1681 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1682 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1683 size=pars['COMPMODELERRFWHMPIVOT'].size)
1684 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1685 size=pars['COMPMODELERRSKYPIVOT'].size)
1686 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1687 size=pars['COMPMODELERRPARS'].size)
1688 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1689 size=pars['COMPEXPGRAY'].size)
1690 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1691 size=pars['COMPVARGRAY'].size)
1692 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1693 doc='Computed exposure offset due to background',
1694 size=pars['COMPEXPDELTAMAGBKG'].size)
1695 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1696 doc='Computed number of good stars per exposure',
1697 size=pars['COMPNGOODSTARPEREXP'].size)
1698 parSchema.addField('compExpRefOffset', type='ArrayD',
1699 doc='Computed per-visit median offset between standard stars and ref stars.',
1700 size=pars['COMPEXPREFOFFSET'].size)
1701 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1702 size=pars['COMPSIGFGCM'].size)
1703 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1704 size=pars['COMPSIGMACAL'].size)
1705 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1706 size=pars['COMPRETRIEVEDLNPWV'].size)
1707 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1708 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1709 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1710 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1711 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1712 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1713 parSchema.addField('compEpsilon', type='ArrayD',
1714 doc='Computed epsilon background offset per visit (nJy/arcsec2)',
1715 size=pars['COMPEPSILON'].size)
1716 parSchema.addField('compMedDeltaAper', type='ArrayD',
1717 doc='Median delta mag aper per visit',
1718 size=pars['COMPMEDDELTAAPER'].size)
1719 parSchema.addField('compGlobalEpsilon', type='ArrayD',
1720 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)',
1721 size=pars['COMPGLOBALEPSILON'].size)
1722 parSchema.addField('compEpsilonMap', type='ArrayD',
1723 doc='Computed epsilon maps (nJy/arcsec2)',
1724 size=pars['COMPEPSILONMAP'].size)
1725 parSchema.addField('compEpsilonNStarMap', type='ArrayI',
1726 doc='Number of stars per pixel in computed epsilon maps',
1727 size=pars['COMPEPSILONNSTARMAP'].size)
1728 parSchema.addField('compEpsilonCcdMap', type='ArrayD',
1729 doc='Computed epsilon ccd maps (nJy/arcsec2)',
1730 size=pars['COMPEPSILONCCDMAP'].size)
1731 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI',
1732 doc='Number of stars per ccd bin in epsilon ccd maps',
1733 size=pars['COMPEPSILONCCDNSTARMAP'].size)
1734 # superstarflat section
1735 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1736 size=4)
1737 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1738 size=parSuperStarFlat.size)
1740 return parSchema
1742 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1743 lutFilterNameString, fitBandString):
1744 """
1745 Make the FGCM parameter catalog for persistence
1747 Parameters
1748 ----------
1749 parSchema: `lsst.afw.table.Schema`
1750 Parameter catalog schema
1751 pars: `numpy.ndarray`
1752 FGCM parameters to put into parCat
1753 parSuperStarFlat: `numpy.array`
1754 FGCM superstar flat array to put into parCat
1755 lutFilterNameString: `str`
1756 Combined string of all the lutFilterNames
1757 fitBandString: `str`
1758 Combined string of all the fitBands
1760 Returns
1761 -------
1762 parCat: `afwTable.BasicCatalog`
1763 Atmosphere and instrumental model parameter catalog for persistence
1764 """
1766 parCat = afwTable.BaseCatalog(parSchema)
1767 parCat.reserve(1)
1769 # The parameter catalog just has one row, with many columns for all the
1770 # atmosphere and instrument fit parameters
1771 rec = parCat.addNew()
1773 # info section
1774 rec['nCcd'] = parInfo['NCCD']
1775 rec['lutFilterNames'] = lutFilterNameString
1776 rec['fitBands'] = fitBandString
1777 # note these are not currently supported here.
1778 rec['hasExternalPwv'] = 0
1779 rec['hasExternalTau'] = 0
1781 # parameter section
1783 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1785 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1786 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1787 'parQeSysIntercept', 'compQeSysSlope',
1788 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1789 'parFilterOffset', 'parFilterOffsetFitFlag',
1790 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1791 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1792 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1793 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1794 'compModelErrSkyPivot', 'compModelErrPars',
1795 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1796 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1797 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1798 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper',
1799 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap',
1800 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap', 'compExpRefOffset']
1802 for scalarName in scalarNames:
1803 rec[scalarName] = pars[scalarName.upper()]
1805 for arrName in arrNames:
1806 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1808 # superstar section
1809 rec['superstarSize'][:] = parSuperStarFlat.shape
1810 rec['superstar'][:] = parSuperStarFlat.ravel()
1812 return parCat
1814 def _makeFlagStarSchema(self):
1815 """
1816 Make the flagged-stars schema
1818 Returns
1819 -------
1820 flagStarSchema: `lsst.afw.table.Schema`
1821 """
1823 flagStarSchema = afwTable.Schema()
1825 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1826 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1828 return flagStarSchema
1830 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1831 """
1832 Make the flagged star catalog for persistence
1834 Parameters
1835 ----------
1836 flagStarSchema: `lsst.afw.table.Schema`
1837 Flagged star schema
1838 flagStarStruct: `numpy.ndarray`
1839 Flagged star structure from fgcm
1841 Returns
1842 -------
1843 flagStarCat: `lsst.afw.table.BaseCatalog`
1844 Flagged star catalog for persistence
1845 """
1847 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1848 flagStarCat.resize(flagStarStruct.size)
1850 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1851 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1853 return flagStarCat