Coverage for python/lsst/fgcmcal/fgcmFitCycle.py: 21%
577 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-01-27 11:24 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-01-27 11:24 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import copy
38import numpy as np
40import lsst.pex.config as pexConfig
41import lsst.pipe.base as pipeBase
42from lsst.pipe.base import connectionTypes
43import lsst.afw.table as afwTable
45from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
46from .utilities import extractReferenceMags
47from .utilities import makeZptSchema, makeZptCat
48from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
49from .sedterms import SedboundarytermDict, SedtermDict
50from .focalPlaneProjector import FocalPlaneProjector
52import fgcm
54__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask']
56MULTIPLE_CYCLES_MAX = 10
59class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument",),
61 defaultTemplates={"previousCycleNumber": "-1",
62 "cycleNumber": "0"}):
63 camera = connectionTypes.PrerequisiteInput(
64 doc="Camera instrument",
65 name="camera",
66 storageClass="Camera",
67 dimensions=("instrument",),
68 isCalibration=True,
69 )
71 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
72 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
73 "chromatic corrections."),
74 name="fgcmLookUpTable",
75 storageClass="Catalog",
76 dimensions=("instrument",),
77 deferLoad=True,
78 )
80 fgcmVisitCatalog = connectionTypes.Input(
81 doc="Catalog of visit information for fgcm",
82 name="fgcmVisitCatalog",
83 storageClass="Catalog",
84 dimensions=("instrument",),
85 deferLoad=True,
86 )
88 fgcmStarObservationsParquet = connectionTypes.Input(
89 doc=("Catalog of star observations for fgcm, in parquet format. "
90 "Used if useParquetCatalogFormat is True."),
91 name="fgcm_star_observations",
92 storageClass="ArrowAstropy",
93 dimensions=("instrument",),
94 deferLoad=True,
95 )
97 fgcmStarIdsParquet = connectionTypes.Input(
98 doc=("Catalog of fgcm calibration star IDs, in parquet format. "
99 "Used if useParquetCatalogFormat is True."),
100 name="fgcm_star_ids",
101 storageClass="ArrowAstropy",
102 dimensions=("instrument",),
103 deferLoad=True,
104 )
106 fgcmReferenceStarsParquet = connectionTypes.Input(
107 doc=("Catalog of fgcm-matched reference stars, in parquet format. "
108 "Used if useParquetCatalogFormat is True."),
109 name="fgcm_reference_stars",
110 storageClass="ArrowAstropy",
111 dimensions=("instrument",),
112 deferLoad=True,
113 )
115 fgcmStarObservations = connectionTypes.Input(
116 doc=("Catalog of star observations for fgcm; old format. "
117 "Used if useParquetCatalogFormat is False."),
118 name="fgcmStarObservations",
119 storageClass="Catalog",
120 dimensions=("instrument",),
121 deferLoad=True,
122 )
124 fgcmStarIds = connectionTypes.Input(
125 doc=("Catalog of fgcm calibration star IDs. "
126 "Used if useParquetCatalogFormat is False."),
127 name="fgcmStarIds",
128 storageClass="Catalog",
129 dimensions=("instrument",),
130 deferLoad=True,
131 )
133 fgcmStarIndices = connectionTypes.Input(
134 doc=("Catalog of fgcm calibration star indices; old format."
135 "Used if useParquetCatalogFormat is False."),
136 name="fgcmStarIndices",
137 storageClass="Catalog",
138 dimensions=("instrument",),
139 deferLoad=True,
140 )
142 fgcmReferenceStars = connectionTypes.Input(
143 doc=("Catalog of fgcm-matched reference stars; old format."
144 "Used if useParquetCatalogFormat is False."),
145 name="fgcmReferenceStars",
146 storageClass="Catalog",
147 dimensions=("instrument",),
148 deferLoad=True,
149 )
151 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
152 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
153 name="fgcmFlaggedStars{previousCycleNumber}",
154 storageClass="Catalog",
155 dimensions=("instrument",),
156 deferLoad=True,
157 )
159 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
160 doc="Catalog of fgcm fit parameters from previous fit cycle",
161 name="fgcmFitParameters{previousCycleNumber}",
162 storageClass="Catalog",
163 dimensions=("instrument",),
164 deferLoad=True,
165 )
167 fgcmFitParameters = connectionTypes.Output(
168 doc="Catalog of fgcm fit parameters from current fit cycle",
169 name="fgcmFitParameters{cycleNumber}",
170 storageClass="Catalog",
171 dimensions=("instrument",),
172 )
174 fgcmFlaggedStars = connectionTypes.Output(
175 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
176 name="fgcmFlaggedStars{cycleNumber}",
177 storageClass="Catalog",
178 dimensions=("instrument",),
179 )
181 fgcmZeropoints = connectionTypes.Output(
182 doc="Catalog of fgcm zeropoint data from current fit cycle",
183 name="fgcmZeropoints{cycleNumber}",
184 storageClass="Catalog",
185 dimensions=("instrument",),
186 )
188 fgcmAtmosphereParameters = connectionTypes.Output(
189 doc="Catalog of atmospheric fit parameters from current fit cycle",
190 name="fgcmAtmosphereParameters{cycleNumber}",
191 storageClass="Catalog",
192 dimensions=("instrument",),
193 )
195 fgcmStandardStars = connectionTypes.Output(
196 doc="Catalog of standard star magnitudes from current fit cycle",
197 name="fgcmStandardStars{cycleNumber}",
198 storageClass="SimpleCatalog",
199 dimensions=("instrument",),
200 )
202 # Add connections for running multiple cycles
203 # This uses vars() to alter the class __dict__ to programmatically
204 # write many similar outputs.
205 for cycle in range(MULTIPLE_CYCLES_MAX):
206 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
207 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
208 name=f"fgcmFitParameters{cycle}",
209 storageClass="Catalog",
210 dimensions=("instrument",),
211 )
212 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
213 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
214 name=f"fgcmFlaggedStars{cycle}",
215 storageClass="Catalog",
216 dimensions=("instrument",),
217 )
218 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
219 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
220 name=f"fgcmZeropoints{cycle}",
221 storageClass="Catalog",
222 dimensions=("instrument",),
223 )
224 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
225 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
226 name=f"fgcmAtmosphereParameters{cycle}",
227 storageClass="Catalog",
228 dimensions=("instrument",),
229 )
230 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
231 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
232 name=f"fgcmStandardStars{cycle}",
233 storageClass="SimpleCatalog",
234 dimensions=("instrument",),
235 )
237 def __init__(self, *, config=None):
238 super().__init__(config=config)
240 if not config.doReferenceCalibration:
241 self.inputs.remove("fgcmReferenceStars")
242 self.inputs.remove("fgcmReferenceStarsParquet")
244 if config.useParquetCatalogFormat:
245 self.inputs.remove("fgcmStarObservations")
246 self.inputs.remove("fgcmStarIds")
247 self.inputs.remove("fgcmStarIndices")
248 if config.doReferenceCalibration:
249 self.inputs.remove("fgcmReferenceStars")
250 else:
251 self.inputs.remove("fgcmStarObservationsParquet")
252 self.inputs.remove("fgcmStarIdsParquet")
253 if config.doReferenceCalibration:
254 self.inputs.remove("fgcmReferenceStarsParquet")
256 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
257 raise ValueError("cycleNumber must be of integer format")
258 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
259 raise ValueError("previousCycleNumber must be of integer format")
260 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
261 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
263 if int(config.connections.cycleNumber) == 0:
264 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
265 self.prerequisiteInputs.remove("fgcmFitParametersInput")
267 if not self.config.doMultipleCycles:
268 # Single-cycle run
269 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
270 self.outputs.remove("fgcmStandardStars")
272 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
273 self.outputs.remove("fgcmZeropoints")
274 self.outputs.remove("fgcmAtmosphereParameters")
276 # Remove all multiple cycle outputs
277 for cycle in range(0, MULTIPLE_CYCLES_MAX):
278 self.outputs.remove(f"fgcmFitParameters{cycle}")
279 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
280 self.outputs.remove(f"fgcmZeropoints{cycle}")
281 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
282 self.outputs.remove(f"fgcmStandardStars{cycle}")
284 else:
285 # Multiple-cycle run
286 # Remove single-cycle outputs
287 self.outputs.remove("fgcmFitParameters")
288 self.outputs.remove("fgcmFlaggedStars")
289 self.outputs.remove("fgcmZeropoints")
290 self.outputs.remove("fgcmAtmosphereParameters")
291 self.outputs.remove("fgcmStandardStars")
293 # Remove outputs from cycles that are not used
294 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
295 MULTIPLE_CYCLES_MAX):
296 self.outputs.remove(f"fgcmFitParameters{cycle}")
297 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
298 self.outputs.remove(f"fgcmZeropoints{cycle}")
299 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
300 self.outputs.remove(f"fgcmStandardStars{cycle}")
302 # Remove non-final-cycle outputs if necessary
303 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
304 if not self.config.outputZeropointsBeforeFinalCycle:
305 self.outputs.remove(f"fgcmZeropoints{cycle}")
306 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
307 if not self.config.outputStandardsBeforeFinalCycle:
308 self.outputs.remove(f"fgcmStandardStars{cycle}")
311class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
312 pipelineConnections=FgcmFitCycleConnections):
313 """Config for FgcmFitCycle"""
315 doMultipleCycles = pexConfig.Field(
316 doc="Run multiple fit cycles in one task",
317 dtype=bool,
318 default=False,
319 )
320 useParquetCatalogFormat = pexConfig.Field(
321 doc="Use parquet catalog format?",
322 dtype=bool,
323 default=True,
324 )
325 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
326 doc=("Final cycle number in multiple cycle mode. The initial cycle "
327 "is 0, with limited parameters fit. The next cycle is 1 with "
328 "full parameter fit. The final cycle is a clean-up with no "
329 "parameters fit. There will be a total of "
330 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
331 "cycle number cannot be less than 2."),
332 dtype=int,
333 default=5,
334 min=2,
335 max=MULTIPLE_CYCLES_MAX,
336 inclusiveMax=True,
337 )
338 bands = pexConfig.ListField(
339 doc="Bands to run calibration",
340 dtype=str,
341 default=[],
342 )
343 fitBands = pexConfig.ListField(
344 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
345 "the atmosphere constrained from the 'fitBands' on the same night. "
346 "Must be a subset of `config.bands`"),
347 dtype=str,
348 default=[],
349 )
350 requiredBands = pexConfig.ListField(
351 doc=("Bands that are required for a star to be considered a calibration star. "
352 "Must be a subset of `config.bands`"),
353 dtype=str,
354 default=[],
355 )
356 physicalFilterMap = pexConfig.DictField(
357 doc="Mapping from 'physicalFilter' to band.",
358 keytype=str,
359 itemtype=str,
360 default={},
361 )
362 doReferenceCalibration = pexConfig.Field(
363 doc="Use reference catalog as additional constraint on calibration",
364 dtype=bool,
365 default=True,
366 )
367 refStarSnMin = pexConfig.Field(
368 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
369 dtype=float,
370 default=50.0,
371 )
372 refStarOutlierNSig = pexConfig.Field(
373 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
374 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
375 dtype=float,
376 default=4.0,
377 )
378 applyRefStarColorCuts = pexConfig.Field(
379 doc=("Apply color cuts defined in ``starColorCuts`` to reference stars? "
380 "These cuts are in addition to any cuts defined in ``refStarColorCuts``"),
381 dtype=bool,
382 default=True,
383 )
384 useExposureReferenceOffset = pexConfig.Field(
385 doc=("Use per-exposure (visit) offsets between calibrated stars and reference stars "
386 "for final zeropoints? This may help uniformity for disjoint surveys."),
387 dtype=bool,
388 default=False,
389 )
390 nCore = pexConfig.Field(
391 doc="Number of cores to use",
392 dtype=int,
393 default=4,
394 )
395 nStarPerRun = pexConfig.Field(
396 doc="Number of stars to run in each chunk",
397 dtype=int,
398 default=200000,
399 )
400 nExpPerRun = pexConfig.Field(
401 doc="Number of exposures to run in each chunk",
402 dtype=int,
403 default=1000,
404 )
405 reserveFraction = pexConfig.Field(
406 doc="Fraction of stars to reserve for testing",
407 dtype=float,
408 default=0.1,
409 )
410 freezeStdAtmosphere = pexConfig.Field(
411 doc="Freeze atmosphere parameters to standard (for testing)",
412 dtype=bool,
413 default=False,
414 )
415 precomputeSuperStarInitialCycle = pexConfig.Field(
416 doc="Precompute superstar flat for initial cycle",
417 dtype=bool,
418 default=False,
419 )
420 superStarSubCcdDict = pexConfig.DictField(
421 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
422 "Must have one entry per band."),
423 keytype=str,
424 itemtype=bool,
425 default={},
426 )
427 superStarSubCcdChebyshevOrder = pexConfig.Field(
428 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
429 "Global default is first-order polynomials, and should be overridden "
430 "on a camera-by-camera basis depending on the ISR."),
431 dtype=int,
432 default=1,
433 )
434 superStarSubCcdTriangular = pexConfig.Field(
435 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
436 "suppress high-order cross terms?"),
437 dtype=bool,
438 default=False,
439 )
440 superStarSigmaClip = pexConfig.Field(
441 doc="Number of sigma to clip outliers when selecting for superstar flats",
442 dtype=float,
443 default=5.0,
444 )
445 superStarPlotCcdResiduals = pexConfig.Field(
446 doc="If plotting is enabled, should per-detector residuals be plotted? "
447 "This may produce a lot of output, and should be used only for "
448 "debugging purposes.",
449 dtype=bool,
450 default=False,
451 )
452 focalPlaneSigmaClip = pexConfig.Field(
453 doc="Number of sigma to clip outliers per focal-plane.",
454 dtype=float,
455 default=4.0,
456 )
457 ccdGraySubCcdDict = pexConfig.DictField(
458 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
459 "('ccd gray') on a sub-ccd scale."),
460 keytype=str,
461 itemtype=bool,
462 default={},
463 )
464 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
465 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
466 dtype=int,
467 default=1,
468 )
469 ccdGraySubCcdTriangular = pexConfig.Field(
470 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
471 "suppress high-order cross terms?"),
472 dtype=bool,
473 default=True,
474 )
475 ccdGrayFocalPlaneDict = pexConfig.DictField(
476 doc=("Per-band specification on whether to compute focal-plane residual "
477 "('ccd gray') corrections."),
478 keytype=str,
479 itemtype=bool,
480 default={},
481 )
482 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
483 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
484 "gray corrections. If there are fewer good CCDs then the gray "
485 "correction is computed per-ccd."),
486 dtype=int,
487 default=1,
488 )
489 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
490 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
491 dtype=int,
492 default=3,
493 )
494 cycleNumber = pexConfig.Field(
495 doc=("FGCM fit cycle number. This is automatically incremented after each run "
496 "and stage of outlier rejection. See cookbook for details."),
497 dtype=int,
498 default=None,
499 )
500 isFinalCycle = pexConfig.Field(
501 doc=("Is this the final cycle of the fitting? Will automatically compute final "
502 "selection of stars and photometric exposures, and will output zeropoints "
503 "and standard stars for use in fgcmOutputProducts"),
504 dtype=bool,
505 default=False,
506 )
507 maxIterBeforeFinalCycle = pexConfig.Field(
508 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
509 "will always be 0 in the final cycle for cleanup and final selection."),
510 dtype=int,
511 default=50,
512 )
513 deltaMagBkgOffsetPercentile = pexConfig.Field(
514 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
515 "offset from local background subtraction."),
516 dtype=float,
517 default=0.25,
518 )
519 deltaMagBkgPerCcd = pexConfig.Field(
520 doc=("Compute net offset from local background subtraction per-ccd? "
521 "Otherwise, use computation per visit."),
522 dtype=bool,
523 default=False,
524 )
525 utBoundary = pexConfig.Field(
526 doc="Boundary (in UTC) from day-to-day",
527 dtype=float,
528 default=None,
529 )
530 washMjds = pexConfig.ListField(
531 doc="Mirror wash MJDs",
532 dtype=float,
533 default=(0.0,),
534 )
535 epochMjds = pexConfig.ListField(
536 doc="Epoch boundaries in MJD",
537 dtype=float,
538 default=(0.0,),
539 )
540 minObsPerBand = pexConfig.Field(
541 doc="Minimum good observations per band",
542 dtype=int,
543 default=2,
544 )
545 # TODO: When DM-16511 is done, it will be possible to get the
546 # telescope latitude directly from the camera.
547 latitude = pexConfig.Field(
548 doc="Observatory latitude",
549 dtype=float,
550 default=None,
551 )
552 mirrorArea = pexConfig.Field(
553 doc="Mirror area (square meters) of telescope. If not set, will "
554 "try to estimate from camera.telescopeDiameter.",
555 dtype=float,
556 default=None,
557 optional=True,
558 )
559 defaultCameraOrientation = pexConfig.Field(
560 doc="Default camera orientation for QA plots.",
561 dtype=float,
562 default=None,
563 )
564 brightObsGrayMax = pexConfig.Field(
565 doc="Maximum gray extinction to be considered bright observation",
566 dtype=float,
567 default=0.15,
568 )
569 minStarPerCcd = pexConfig.Field(
570 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
571 "CCDs with fewer stars will have their calibration estimated from other "
572 "CCDs in the same visit, with zeropoint error increased accordingly."),
573 dtype=int,
574 default=5,
575 )
576 minCcdPerExp = pexConfig.Field(
577 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
578 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
579 dtype=int,
580 default=5,
581 )
582 maxCcdGrayErr = pexConfig.Field(
583 doc="Maximum error on CCD gray offset to be considered photometric",
584 dtype=float,
585 default=0.05,
586 )
587 minStarPerExp = pexConfig.Field(
588 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
589 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
590 dtype=int,
591 default=600,
592 )
593 minExpPerNight = pexConfig.Field(
594 doc="Minimum number of good exposures/visits to consider a partly photometric night",
595 dtype=int,
596 default=10,
597 )
598 expGrayInitialCut = pexConfig.Field(
599 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
600 "observations."),
601 dtype=float,
602 default=-0.25,
603 )
604 expGrayPhotometricCutDict = pexConfig.DictField(
605 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
606 "('gray term') for a visit to be considered photometric. Must have one "
607 "entry per band. Broad-band filters should be -0.05."),
608 keytype=str,
609 itemtype=float,
610 default={},
611 )
612 expGrayHighCutDict = pexConfig.DictField(
613 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
614 "('gray term') for a visit to be considered photometric. Must have one "
615 "entry per band. Broad-band filters should be 0.2."),
616 keytype=str,
617 itemtype=float,
618 default={},
619 )
620 expGrayRecoverCut = pexConfig.Field(
621 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
622 "Visits with more gray extinction will only get CCD zeropoints if there are "
623 "sufficient star observations (minStarPerCcd) on that CCD."),
624 dtype=float,
625 default=-1.0,
626 )
627 expVarGrayPhotometricCutDict = pexConfig.DictField(
628 doc=("Per-band specification on maximum exposure variance to be considered possibly "
629 "photometric. Must have one entry per band. Broad-band filters should be "
630 "0.0005."),
631 keytype=str,
632 itemtype=float,
633 default={},
634 )
635 expGrayErrRecoverCut = pexConfig.Field(
636 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
637 "Visits with more gray variance will only get CCD zeropoints if there are "
638 "sufficient star observations (minStarPerCcd) on that CCD."),
639 dtype=float,
640 default=0.05,
641 )
642 aperCorrFitNBins = pexConfig.Field(
643 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
644 "no fit will be performed, and the config.aperCorrInputSlopes will be "
645 "used if available."),
646 dtype=int,
647 default=10,
648 )
649 aperCorrInputSlopeDict = pexConfig.DictField(
650 doc=("Per-band specification of aperture correction input slope parameters. These "
651 "are used on the first fit iteration, and aperture correction parameters will "
652 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
653 "to set this when there is insufficient data to fit the parameters (e.g. "
654 "tract mode)."),
655 keytype=str,
656 itemtype=float,
657 default={},
658 )
659 sedboundaryterms = pexConfig.ConfigField(
660 doc="Mapping from bands to SED boundary term names used is sedterms.",
661 dtype=SedboundarytermDict,
662 )
663 sedterms = pexConfig.ConfigField(
664 doc="Mapping from terms to bands for fgcm linear SED approximations.",
665 dtype=SedtermDict,
666 )
667 sigFgcmMaxErr = pexConfig.Field(
668 doc="Maximum mag error for fitting sigma_FGCM",
669 dtype=float,
670 default=0.01,
671 )
672 sigFgcmMaxEGrayDict = pexConfig.DictField(
673 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
674 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
675 "should be 0.05."),
676 keytype=str,
677 itemtype=float,
678 default={},
679 )
680 ccdGrayMaxStarErr = pexConfig.Field(
681 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
682 "computation"),
683 dtype=float,
684 default=0.10,
685 )
686 approxThroughputDict = pexConfig.DictField(
687 doc=("Per-band specification of the approximate overall throughput at the start of "
688 "calibration observations. Must have one entry per band. Typically should "
689 "be 1.0."),
690 keytype=str,
691 itemtype=float,
692 default={},
693 )
694 sigmaCalRange = pexConfig.ListField(
695 doc="Allowed range for systematic error floor estimation",
696 dtype=float,
697 default=(0.001, 0.003),
698 )
699 sigmaCalFitPercentile = pexConfig.ListField(
700 doc="Magnitude percentile range to fit systematic error floor",
701 dtype=float,
702 default=(0.05, 0.15),
703 )
704 sigmaCalPlotPercentile = pexConfig.ListField(
705 doc="Magnitude percentile range to plot systematic error floor",
706 dtype=float,
707 default=(0.05, 0.95),
708 )
709 sigma0Phot = pexConfig.Field(
710 doc="Systematic error floor for all zeropoints",
711 dtype=float,
712 default=0.003,
713 )
714 mapLongitudeRef = pexConfig.Field(
715 doc="Reference longitude for plotting maps",
716 dtype=float,
717 default=0.0,
718 )
719 mapNSide = pexConfig.Field(
720 doc="Healpix nside for plotting maps",
721 dtype=int,
722 default=256,
723 )
724 outfileBase = pexConfig.Field(
725 doc="Filename start for plot output files",
726 dtype=str,
727 default=None,
728 )
729 starColorCuts = pexConfig.ListField(
730 doc=("Encoded star-color cuts (using calibration star colors). "
731 "This is a list with each entry a string of the format "
732 "``band1,band2,low,high`` such that only stars of color "
733 "low < band1 - band2 < high will be used for calibration."),
734 dtype=str,
735 default=("NO_DATA",),
736 )
737 refStarColorCuts = pexConfig.ListField(
738 doc=("Encoded star color cuts specifically to apply to reference stars. "
739 "This is a list with each entry a string of the format "
740 "``band1,band2,low,high`` such that only stars of color "
741 "low < band1 - band2 < high will be used as reference stars."),
742 dtype=str,
743 default=("NO_DATA",),
744 )
745 colorSplitBands = pexConfig.ListField(
746 doc="Band names to use to split stars by color. Must have 2 entries.",
747 dtype=str,
748 length=2,
749 default=('g', 'i'),
750 )
751 modelMagErrors = pexConfig.Field(
752 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
753 dtype=bool,
754 default=True,
755 )
756 useQuadraticPwv = pexConfig.Field(
757 doc="Model PWV with a quadratic term for variation through the night?",
758 dtype=bool,
759 default=False,
760 )
761 instrumentParsPerBand = pexConfig.Field(
762 doc=("Model instrumental parameters per band? "
763 "Otherwise, instrumental parameters (QE changes with time) are "
764 "shared among all bands."),
765 dtype=bool,
766 default=False,
767 )
768 instrumentSlopeMinDeltaT = pexConfig.Field(
769 doc=("Minimum time change (in days) between observations to use in constraining "
770 "instrument slope."),
771 dtype=float,
772 default=20.0,
773 )
774 fitMirrorChromaticity = pexConfig.Field(
775 doc="Fit (intraband) mirror chromatic term?",
776 dtype=bool,
777 default=False,
778 )
779 fitCcdChromaticityDict = pexConfig.DictField(
780 doc="Specification on whether to compute first-order quantum efficiency (QE) "
781 "adjustments. Key is band, and value will be True or False. Any band "
782 "not explicitly specified will default to False.",
783 keytype=str,
784 itemtype=bool,
785 default={},
786 )
787 coatingMjds = pexConfig.ListField(
788 doc="Mirror coating dates in MJD",
789 dtype=float,
790 default=(0.0,),
791 )
792 outputStandardsBeforeFinalCycle = pexConfig.Field(
793 doc="Output standard stars prior to final cycle? Used in debugging.",
794 dtype=bool,
795 default=False,
796 )
797 outputZeropointsBeforeFinalCycle = pexConfig.Field(
798 doc="Output standard stars prior to final cycle? Used in debugging.",
799 dtype=bool,
800 default=False,
801 )
802 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
803 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
804 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
805 keytype=str,
806 itemtype=bool,
807 default={},
808 )
809 autoPhotometricCutNSig = pexConfig.Field(
810 doc=("Number of sigma for automatic computation of (low) photometric cut. "
811 "Cut is based on exposure gray width (per band), unless "
812 "useRepeatabilityForExpGrayCuts is set, in which case the star "
813 "repeatability is used (also per band)."),
814 dtype=float,
815 default=3.0,
816 )
817 autoHighCutNSig = pexConfig.Field(
818 doc=("Number of sigma for automatic computation of (high) outlier cut. "
819 "Cut is based on exposure gray width (per band), unless "
820 "useRepeatabilityForExpGrayCuts is set, in which case the star "
821 "repeatability is used (also per band)."),
822 dtype=float,
823 default=4.0,
824 )
825 quietMode = pexConfig.Field(
826 doc="Be less verbose with logging.",
827 dtype=bool,
828 default=False,
829 )
830 doPlots = pexConfig.Field(
831 doc="Make fgcm QA plots.",
832 dtype=bool,
833 default=True,
834 )
835 randomSeed = pexConfig.Field(
836 doc="Random seed for fgcm for consistency in tests.",
837 dtype=int,
838 default=None,
839 optional=True,
840 )
841 deltaAperFitMinNgoodObs = pexConfig.Field(
842 doc="Minimum number of good observations to use mean delta-aper values in fits.",
843 dtype=int,
844 default=2,
845 )
846 deltaAperFitPerCcdNx = pexConfig.Field(
847 doc=("Number of x bins per ccd when computing delta-aper background offsets. "
848 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
849 dtype=int,
850 default=10,
851 )
852 deltaAperFitPerCcdNy = pexConfig.Field(
853 doc=("Number of y bins per ccd when computing delta-aper background offsets. "
854 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
855 dtype=int,
856 default=10,
857 )
858 deltaAperFitSpatialNside = pexConfig.Field(
859 doc="Healpix nside to compute spatial delta-aper background offset maps.",
860 dtype=int,
861 default=64,
862 )
863 deltaAperInnerRadiusArcsec = pexConfig.Field(
864 doc=("Inner radius used to compute deltaMagAper (arcseconds). "
865 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if "
866 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
867 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
868 dtype=float,
869 default=0.0,
870 )
871 deltaAperOuterRadiusArcsec = pexConfig.Field(
872 doc=("Outer radius used to compute deltaMagAper (arcseconds). "
873 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if "
874 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
875 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
876 dtype=float,
877 default=0.0,
878 )
879 doComputeDeltaAperPerVisit = pexConfig.Field(
880 doc=("Do the computation of delta-aper background offsets per visit? "
881 "Note: this option can be very slow when there are many visits."),
882 dtype=bool,
883 default=False,
884 )
885 doComputeDeltaAperPerStar = pexConfig.Field(
886 doc="Do the computation of delta-aper mean values per star?",
887 dtype=bool,
888 default=True,
889 )
890 doComputeDeltaAperMap = pexConfig.Field(
891 doc=("Do the computation of delta-aper spatial maps? "
892 "This is only used if ``doComputeDeltaAperPerStar`` is True,"),
893 dtype=bool,
894 default=False,
895 )
896 doComputeDeltaAperPerCcd = pexConfig.Field(
897 doc="Do the computation of per-ccd delta-aper background offsets?",
898 dtype=bool,
899 default=False,
900 )
902 def validate(self):
903 super().validate()
905 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
906 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
907 raise RuntimeError(msg)
908 if self.connections.cycleNumber != str(self.cycleNumber):
909 msg = "cycleNumber in template must be equal to connections.cycleNumber"
910 raise RuntimeError(msg)
912 for band in self.fitBands:
913 if band not in self.bands:
914 msg = 'fitBand %s not in bands' % (band)
915 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
916 for band in self.requiredBands:
917 if band not in self.bands:
918 msg = 'requiredBand %s not in bands' % (band)
919 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
920 for band in self.colorSplitBands:
921 if band not in self.bands:
922 msg = 'colorSplitBand %s not in bands' % (band)
923 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
924 for band in self.bands:
925 if band not in self.superStarSubCcdDict:
926 msg = 'band %s not in superStarSubCcdDict' % (band)
927 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
928 self, msg)
929 if band not in self.ccdGraySubCcdDict:
930 msg = 'band %s not in ccdGraySubCcdDict' % (band)
931 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
932 self, msg)
933 if band not in self.expGrayPhotometricCutDict:
934 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
935 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
936 self, msg)
937 if band not in self.expGrayHighCutDict:
938 msg = 'band %s not in expGrayHighCutDict' % (band)
939 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
940 self, msg)
941 if band not in self.expVarGrayPhotometricCutDict:
942 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
943 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
944 self, msg)
945 if band not in self.sigFgcmMaxEGrayDict:
946 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
947 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
948 self, msg)
949 if band not in self.approxThroughputDict:
950 msg = 'band %s not in approxThroughputDict' % (band)
951 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
952 self, msg)
953 if band not in self.useRepeatabilityForExpGrayCutsDict:
954 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
955 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
956 self, msg)
958 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \
959 or self.doComputeDeltaAperPerCcd:
960 if self.deltaAperInnerRadiusArcsec <= 0.0:
961 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.'
962 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec,
963 self, msg)
964 if self.deltaAperOuterRadiusArcsec <= 0.0:
965 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.'
966 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
967 self, msg)
968 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec:
969 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if '
970 'deltaAper computations are turned on.')
971 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
972 self, msg)
975class FgcmFitCycleTask(pipeBase.PipelineTask):
976 """
977 Run Single fit cycle for FGCM global calibration
978 """
980 ConfigClass = FgcmFitCycleConfig
981 _DefaultName = "fgcmFitCycle"
983 def __init__(self, initInputs=None, **kwargs):
984 super().__init__(**kwargs)
986 def runQuantum(self, butlerQC, inputRefs, outputRefs):
987 camera = butlerQC.get(inputRefs.camera)
989 handleDict = {}
991 handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
992 handleDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
994 if self.config.useParquetCatalogFormat:
995 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservationsParquet)
996 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIdsParquet)
997 if self.config.doReferenceCalibration:
998 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStarsParquet)
999 else:
1000 handleDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
1001 handleDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
1002 handleDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
1003 if self.config.doReferenceCalibration:
1004 handleDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
1005 if self.config.cycleNumber > 0:
1006 handleDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
1007 handleDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
1009 fgcmDatasetDict = None
1010 if self.config.doMultipleCycles:
1011 # Run multiple cycles at once.
1012 config = copy.copy(self.config)
1013 config.update(cycleNumber=0)
1014 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
1015 if cycle == self.config.multipleCyclesFinalCycleNumber:
1016 config.update(isFinalCycle=True)
1018 if cycle > 0:
1019 handleDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
1020 handleDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
1022 fgcmDatasetDict, config = self._fgcmFitCycle(camera, handleDict, config=config)
1023 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
1024 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
1025 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
1026 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
1027 if self.outputZeropoints:
1028 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
1029 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
1030 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
1031 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
1032 if self.outputStandards:
1033 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
1034 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
1035 else:
1036 # Run a single cycle
1037 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, handleDict)
1039 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
1040 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
1041 if self.outputZeropoints:
1042 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
1043 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
1044 if self.outputStandards:
1045 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
1047 def _fgcmFitCycle(self, camera, handleDict, config=None):
1048 """
1049 Run the fit cycle
1051 Parameters
1052 ----------
1053 camera : `lsst.afw.cameraGeom.Camera`
1054 handleDict : `dict`
1055 All handles are `lsst.daf.butler.DeferredDatasetHandle`
1056 handle dictionary with keys:
1058 ``"fgcmLookUpTable"``
1059 handle for the FGCM look-up table.
1060 ``"fgcmVisitCatalog"``
1061 handle for visit summary catalog.
1062 ``"fgcmStarObservations"``
1063 handle for star observation catalog.
1064 ``"fgcmStarIds"``
1065 handle for star id catalog.
1066 ``"fgcmStarIndices"``
1067 handle for star index catalog.
1068 ``"fgcmReferenceStars"``
1069 handle for matched reference star catalog.
1070 ``"fgcmFlaggedStars"``
1071 handle for flagged star catalog.
1072 ``"fgcmFitParameters"``
1073 handle for fit parameter catalog.
1074 config : `lsst.pex.config.Config`, optional
1075 Configuration to use to override self.config.
1077 Returns
1078 -------
1079 fgcmDatasetDict : `dict`
1080 Dictionary of datasets to persist.
1081 """
1082 if config is not None:
1083 _config = config
1084 else:
1085 _config = self.config
1087 # Set defaults on whether to output standards and zeropoints
1088 self.maxIter = _config.maxIterBeforeFinalCycle
1089 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1090 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1091 self.resetFitParameters = True
1093 if _config.isFinalCycle:
1094 # This is the final fit cycle, so we do not want to reset fit
1095 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1096 # and we always want to output standards and zeropoints
1097 self.maxIter = 0
1098 self.outputStandards = True
1099 self.outputZeropoints = True
1100 self.resetFitParameters = False
1102 lutCat = handleDict['fgcmLookUpTable'].get()
1103 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1104 dict(_config.physicalFilterMap))
1105 del lutCat
1107 configDict = makeConfigDict(_config, self.log, camera,
1108 self.maxIter, self.resetFitParameters,
1109 self.outputZeropoints,
1110 lutIndexVals[0]['FILTERNAMES'])
1112 # next we need the exposure/visit information
1113 visitCat = handleDict['fgcmVisitCatalog'].get()
1114 fgcmExpInfo = translateVisitCatalog(visitCat)
1115 del visitCat
1117 focalPlaneProjector = FocalPlaneProjector(camera,
1118 self.config.defaultCameraOrientation)
1120 noFitsDict = {'lutIndex': lutIndexVals,
1121 'lutStd': lutStd,
1122 'expInfo': fgcmExpInfo,
1123 'focalPlaneProjector': focalPlaneProjector}
1125 # set up the fitter object
1126 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1127 noFitsDict=noFitsDict, noOutput=True)
1129 # create the parameter object
1130 if (fgcmFitCycle.initialCycle):
1131 # cycle = 0, initial cycle
1132 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1133 fgcmLut,
1134 fgcmExpInfo)
1135 else:
1136 if isinstance(handleDict['fgcmFitParameters'], afwTable.BaseCatalog):
1137 parCat = handleDict['fgcmFitParameters']
1138 else:
1139 parCat = handleDict['fgcmFitParameters'].get()
1140 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1141 del parCat
1142 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1143 fgcmExpInfo,
1144 inParInfo,
1145 inParams,
1146 inSuperStar)
1148 # set up the stars...
1149 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1151 starObs = handleDict['fgcmStarObservations'].get()
1152 starIds = handleDict['fgcmStarIds'].get()
1153 if not self.config.useParquetCatalogFormat:
1154 starIndices = handleDict['fgcmStarIndices'].get()
1155 else:
1156 starIndices = None
1158 # grab the flagged stars if available
1159 if 'fgcmFlaggedStars' in handleDict:
1160 if isinstance(handleDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1161 flaggedStars = handleDict['fgcmFlaggedStars']
1162 else:
1163 flaggedStars = handleDict['fgcmFlaggedStars'].get()
1164 flagId = flaggedStars['objId'][:]
1165 flagFlag = flaggedStars['objFlag'][:]
1167 del flaggedStars
1168 elif self.config.useParquetCatalogFormat:
1169 # If we are using the parquet catalog format, then that means that
1170 # reserved stars have already been flagged. We extract the flags here
1171 # to input to fgcm, which will then be persisted (with additional
1172 # quality flags) as the fgcmFlaggedStars datatype in subsequent
1173 # fit cycles.
1174 (flagged,) = (starIds['obj_flag'] > 0).nonzero()
1175 flagId = starIds['fgcm_id'][flagged]
1176 flagFlag = starIds['obj_flag'][flagged]
1177 else:
1178 flagId = None
1179 flagFlag = None
1181 if _config.doReferenceCalibration:
1182 refStars = handleDict['fgcmReferenceStars'].get()
1184 refMag, refMagErr = extractReferenceMags(refStars,
1185 _config.bands,
1186 _config.physicalFilterMap)
1188 refId = refStars['fgcm_id'][:]
1189 else:
1190 refStars = None
1191 refId = None
1192 refMag = None
1193 refMagErr = None
1195 # match star observations to visits
1196 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1197 # actually be transferred into fgcm using the indexing below.
1198 if self.config.useParquetCatalogFormat:
1199 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'])
1200 else:
1201 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1203 # The fgcmStars.loadStars method will copy all the star information into
1204 # special shared memory objects that will not blow up the memory usage when
1205 # used with python multiprocessing. Once all the numbers are copied,
1206 # it is necessary to release all references to the objects that previously
1207 # stored the data to ensure that the garbage collector can clear the memory,
1208 # and ensure that this memory is not copied when multiprocessing kicks in.
1210 if self.config.useParquetCatalogFormat:
1211 # Note that the ra/dec coordinates for the parquet format are in
1212 # degrees, which is what fgcm expects.
1213 fgcmStars.loadStars(fgcmPars,
1214 starObs['visit'][:],
1215 starObs['detector'][:],
1216 starObs['ra'][:],
1217 starObs['dec'][:],
1218 starObs['inst_mag'][:],
1219 starObs['inst_mag_err'][:],
1220 fgcmExpInfo['FILTERNAME'][visitIndex],
1221 starIds['fgcm_id'][:],
1222 starIds['ra'][:],
1223 starIds['dec'][:],
1224 starIds['obs_arr_index'][:],
1225 starIds['n_obs'][:],
1226 obsX=starObs['x'][:],
1227 obsY=starObs['y'][:],
1228 obsDeltaMagBkg=starObs['delta_mag_bkg'][:],
1229 obsDeltaAper=starObs['delta_mag_aper'][:],
1230 refID=refId,
1231 refMag=refMag,
1232 refMagErr=refMagErr,
1233 flagID=flagId,
1234 flagFlag=flagFlag,
1235 computeNobs=True)
1236 else:
1237 # We determine the conversion from the native units (typically radians) to
1238 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1239 # numpy arrays rather than Angles, which would we approximately 600x slower.
1240 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1242 fgcmStars.loadStars(fgcmPars,
1243 starObs['visit'][starIndices['obsIndex']],
1244 starObs['ccd'][starIndices['obsIndex']],
1245 starObs['ra'][starIndices['obsIndex']] * conv,
1246 starObs['dec'][starIndices['obsIndex']] * conv,
1247 starObs['instMag'][starIndices['obsIndex']],
1248 starObs['instMagErr'][starIndices['obsIndex']],
1249 fgcmExpInfo['FILTERNAME'][visitIndex],
1250 starIds['fgcm_id'][:],
1251 starIds['ra'][:],
1252 starIds['dec'][:],
1253 starIds['obsArrIndex'][:],
1254 starIds['nObs'][:],
1255 obsX=starObs['x'][starIndices['obsIndex']],
1256 obsY=starObs['y'][starIndices['obsIndex']],
1257 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1258 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']],
1259 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1260 refID=refId,
1261 refMag=refMag,
1262 refMagErr=refMagErr,
1263 flagID=flagId,
1264 flagFlag=flagFlag,
1265 computeNobs=True)
1267 # Release all references to temporary objects holding star data (see above)
1268 del starObs
1269 del starIds
1270 del starIndices
1271 del flagId
1272 del flagFlag
1273 del refStars
1274 del refId
1275 del refMag
1276 del refMagErr
1278 # and set the bits in the cycle object
1279 fgcmFitCycle.setLUT(fgcmLut)
1280 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1281 fgcmFitCycle.setPars(fgcmPars)
1283 # finish the setup
1284 fgcmFitCycle.finishSetup()
1286 # and run
1287 fgcmFitCycle.run()
1289 ##################
1290 # Persistance
1291 ##################
1293 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1295 # Output the config for the next cycle
1296 # We need to make a copy since the input one has been frozen
1298 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1299 i, b in enumerate(_config.bands)}
1300 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1301 i, band in enumerate(_config.bands)}
1303 outConfig = copy.copy(_config)
1304 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1305 precomputeSuperStarInitialCycle=False,
1306 freezeStdAtmosphere=False,
1307 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1308 expGrayHighCutDict=updatedHighCutDict)
1310 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1311 cycleNumber=str(_config.cycleNumber + 1))
1313 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1314 outConfig.cycleNumber)
1315 outConfig.save(configFileName)
1317 if _config.isFinalCycle == 1:
1318 # We are done, ready to output products
1319 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1320 else:
1321 self.log.info("Saved config for next cycle to %s" % (configFileName))
1322 self.log.info("Be sure to look at:")
1323 self.log.info(" config.expGrayPhotometricCut")
1324 self.log.info(" config.expGrayHighCut")
1325 self.log.info("If you are satisfied with the fit, please set:")
1326 self.log.info(" config.isFinalCycle = True")
1328 fgcmFitCycle.freeSharedMemory()
1330 return fgcmDatasetDict, outConfig
1332 def _loadParameters(self, parCat):
1333 """
1334 Load FGCM parameters from a previous fit cycle
1336 Parameters
1337 ----------
1338 parCat : `lsst.afw.table.BaseCatalog`
1339 Parameter catalog in afw table form.
1341 Returns
1342 -------
1343 inParInfo: `numpy.ndarray`
1344 Numpy array parameter information formatted for input to fgcm
1345 inParameters: `numpy.ndarray`
1346 Numpy array parameter values formatted for input to fgcm
1347 inSuperStar: `numpy.array`
1348 Superstar flat formatted for input to fgcm
1349 """
1350 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1351 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1353 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1354 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1355 (parLutFilterNames.size, )),
1356 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1357 ('LNTAUUNIT', 'f8'),
1358 ('LNTAUSLOPEUNIT', 'f8'),
1359 ('ALPHAUNIT', 'f8'),
1360 ('LNPWVUNIT', 'f8'),
1361 ('LNPWVSLOPEUNIT', 'f8'),
1362 ('LNPWVQUADRATICUNIT', 'f8'),
1363 ('LNPWVGLOBALUNIT', 'f8'),
1364 ('O3UNIT', 'f8'),
1365 ('QESYSUNIT', 'f8'),
1366 ('FILTEROFFSETUNIT', 'f8'),
1367 ('HASEXTERNALPWV', 'i2'),
1368 ('HASEXTERNALTAU', 'i2')])
1369 inParInfo['NCCD'] = parCat['nCcd']
1370 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1371 inParInfo['FITBANDS'][:] = parFitBands
1372 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1373 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1375 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1376 ('PARO3', 'f8', (parCat['parO3'].size, )),
1377 ('PARLNTAUINTERCEPT', 'f8',
1378 (parCat['parLnTauIntercept'].size, )),
1379 ('PARLNTAUSLOPE', 'f8',
1380 (parCat['parLnTauSlope'].size, )),
1381 ('PARLNPWVINTERCEPT', 'f8',
1382 (parCat['parLnPwvIntercept'].size, )),
1383 ('PARLNPWVSLOPE', 'f8',
1384 (parCat['parLnPwvSlope'].size, )),
1385 ('PARLNPWVQUADRATIC', 'f8',
1386 (parCat['parLnPwvQuadratic'].size, )),
1387 ('PARQESYSINTERCEPT', 'f8',
1388 (parCat['parQeSysIntercept'].size, )),
1389 ('COMPQESYSSLOPE', 'f8',
1390 (parCat['compQeSysSlope'].size, )),
1391 ('PARFILTEROFFSET', 'f8',
1392 (parCat['parFilterOffset'].size, )),
1393 ('PARFILTEROFFSETFITFLAG', 'i2',
1394 (parCat['parFilterOffsetFitFlag'].size, )),
1395 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1396 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1397 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1398 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1399 ('COMPABSTHROUGHPUT', 'f8',
1400 (parCat['compAbsThroughput'].size, )),
1401 ('COMPREFOFFSET', 'f8',
1402 (parCat['compRefOffset'].size, )),
1403 ('COMPREFSIGMA', 'f8',
1404 (parCat['compRefSigma'].size, )),
1405 ('COMPMIRRORCHROMATICITY', 'f8',
1406 (parCat['compMirrorChromaticity'].size, )),
1407 ('MIRRORCHROMATICITYPIVOT', 'f8',
1408 (parCat['mirrorChromaticityPivot'].size, )),
1409 ('COMPCCDCHROMATICITY', 'f8',
1410 (parCat['compCcdChromaticity'].size, )),
1411 ('COMPMEDIANSEDSLOPE', 'f8',
1412 (parCat['compMedianSedSlope'].size, )),
1413 ('COMPAPERCORRPIVOT', 'f8',
1414 (parCat['compAperCorrPivot'].size, )),
1415 ('COMPAPERCORRSLOPE', 'f8',
1416 (parCat['compAperCorrSlope'].size, )),
1417 ('COMPAPERCORRSLOPEERR', 'f8',
1418 (parCat['compAperCorrSlopeErr'].size, )),
1419 ('COMPAPERCORRRANGE', 'f8',
1420 (parCat['compAperCorrRange'].size, )),
1421 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1422 (parCat['compModelErrExptimePivot'].size, )),
1423 ('COMPMODELERRFWHMPIVOT', 'f8',
1424 (parCat['compModelErrFwhmPivot'].size, )),
1425 ('COMPMODELERRSKYPIVOT', 'f8',
1426 (parCat['compModelErrSkyPivot'].size, )),
1427 ('COMPMODELERRPARS', 'f8',
1428 (parCat['compModelErrPars'].size, )),
1429 ('COMPEXPGRAY', 'f8',
1430 (parCat['compExpGray'].size, )),
1431 ('COMPVARGRAY', 'f8',
1432 (parCat['compVarGray'].size, )),
1433 ('COMPEXPDELTAMAGBKG', 'f8',
1434 (parCat['compExpDeltaMagBkg'].size, )),
1435 ('COMPNGOODSTARPEREXP', 'i4',
1436 (parCat['compNGoodStarPerExp'].size, )),
1437 ('COMPEXPREFOFFSET', 'f8',
1438 (parCat['compExpRefOffset'].size, )),
1439 ('COMPSIGFGCM', 'f8',
1440 (parCat['compSigFgcm'].size, )),
1441 ('COMPSIGMACAL', 'f8',
1442 (parCat['compSigmaCal'].size, )),
1443 ('COMPRETRIEVEDLNPWV', 'f8',
1444 (parCat['compRetrievedLnPwv'].size, )),
1445 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1446 (parCat['compRetrievedLnPwvRaw'].size, )),
1447 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1448 (parCat['compRetrievedLnPwvFlag'].size, )),
1449 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1450 (parCat['compRetrievedTauNight'].size, )),
1451 ('COMPEPSILON', 'f8',
1452 (parCat['compEpsilon'].size, )),
1453 ('COMPMEDDELTAAPER', 'f8',
1454 (parCat['compMedDeltaAper'].size, )),
1455 ('COMPGLOBALEPSILON', 'f4',
1456 (parCat['compGlobalEpsilon'].size, )),
1457 ('COMPEPSILONMAP', 'f4',
1458 (parCat['compEpsilonMap'].size, )),
1459 ('COMPEPSILONNSTARMAP', 'i4',
1460 (parCat['compEpsilonNStarMap'].size, )),
1461 ('COMPEPSILONCCDMAP', 'f4',
1462 (parCat['compEpsilonCcdMap'].size, )),
1463 ('COMPEPSILONCCDNSTARMAP', 'i4',
1464 (parCat['compEpsilonCcdNStarMap'].size, ))])
1466 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1467 inParams['PARO3'][:] = parCat['parO3'][0, :]
1468 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1469 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1470 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1471 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1472 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1473 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1474 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1475 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1476 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1477 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1478 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1479 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1480 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1481 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1482 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1483 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1484 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1485 inParams['COMPCCDCHROMATICITY'][:] = parCat['compCcdChromaticity'][0, :]
1486 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1487 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1488 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1489 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1490 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1491 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1492 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1493 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1494 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1495 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1496 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1497 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1498 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1499 inParams['COMPEXPREFOFFSET'][:] = parCat['compExpRefOffset'][0, :]
1500 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1501 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1502 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1503 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1504 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1505 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1506 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :]
1507 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :]
1508 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :]
1509 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :]
1510 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :]
1511 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :]
1512 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :]
1514 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1515 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1517 return (inParInfo, inParams, inSuperStar)
1519 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1520 """
1521 Persist FGCM datasets through the butler.
1523 Parameters
1524 ----------
1525 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1526 Fgcm Fit cycle object
1527 """
1528 fgcmDatasetDict = {}
1530 # Save the parameters
1531 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1533 parSchema = afwTable.Schema()
1535 comma = ','
1536 lutFilterNameString = comma.join([n.decode('utf-8')
1537 for n in parInfo['LUTFILTERNAMES'][0]])
1538 fitBandString = comma.join([n.decode('utf-8')
1539 for n in parInfo['FITBANDS'][0]])
1541 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1542 lutFilterNameString, fitBandString)
1543 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1544 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1545 lutFilterNameString, fitBandString)
1547 fgcmDatasetDict['fgcmFitParameters'] = parCat
1549 # Save the indices of the flagged stars
1550 # (stars that have been (a) reserved from the fit for testing and
1551 # (b) bad stars that have failed quality checks.)
1552 flagStarSchema = self._makeFlagStarSchema()
1553 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1554 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1556 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1558 # Save the zeropoint information and atmospheres only if desired
1559 if self.outputZeropoints:
1560 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1561 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1563 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1564 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1566 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1568 # Save atmosphere values
1569 # These are generated by the same code that generates zeropoints
1570 atmSchema = makeAtmSchema()
1571 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1573 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1575 # Save the standard stars (if configured)
1576 if self.outputStandards:
1577 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1578 stdSchema = makeStdSchema(len(goodBands))
1579 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1581 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1583 return fgcmDatasetDict
1585 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1586 lutFilterNameString, fitBandString):
1587 """
1588 Make the parameter persistence schema
1590 Parameters
1591 ----------
1592 parInfo: `numpy.ndarray`
1593 Parameter information returned by fgcm
1594 pars: `numpy.ndarray`
1595 Parameter values returned by fgcm
1596 parSuperStarFlat: `numpy.array`
1597 Superstar flat values returned by fgcm
1598 lutFilterNameString: `str`
1599 Combined string of all the lutFilterNames
1600 fitBandString: `str`
1601 Combined string of all the fitBands
1603 Returns
1604 -------
1605 parSchema: `afwTable.schema`
1606 """
1608 parSchema = afwTable.Schema()
1610 # parameter info section
1611 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1612 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1613 size=len(lutFilterNameString))
1614 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1615 size=len(fitBandString))
1616 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1617 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1618 doc='Step units for ln(AOD) slope')
1619 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1620 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1621 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1622 doc='Step units for ln(pwv) slope')
1623 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1624 doc='Step units for ln(pwv) quadratic term')
1625 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1626 doc='Step units for global ln(pwv) parameters')
1627 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1628 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1629 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1630 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1631 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1633 # parameter section
1634 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1635 size=pars['PARALPHA'].size)
1636 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1637 size=pars['PARO3'].size)
1638 parSchema.addField('parLnTauIntercept', type='ArrayD',
1639 doc='ln(Tau) intercept parameter vector',
1640 size=pars['PARLNTAUINTERCEPT'].size)
1641 parSchema.addField('parLnTauSlope', type='ArrayD',
1642 doc='ln(Tau) slope parameter vector',
1643 size=pars['PARLNTAUSLOPE'].size)
1644 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1645 size=pars['PARLNPWVINTERCEPT'].size)
1646 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1647 size=pars['PARLNPWVSLOPE'].size)
1648 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1649 size=pars['PARLNPWVQUADRATIC'].size)
1650 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1651 size=pars['PARQESYSINTERCEPT'].size)
1652 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1653 size=pars[0]['COMPQESYSSLOPE'].size)
1654 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1655 size=pars['PARFILTEROFFSET'].size)
1656 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1657 size=pars['PARFILTEROFFSETFITFLAG'].size)
1658 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1659 doc='Global scale for retrieved ln(pwv)')
1660 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1661 doc='Global offset for retrieved ln(pwv)')
1662 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1663 doc='Nightly offset for retrieved ln(pwv)',
1664 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1665 parSchema.addField('compAbsThroughput', type='ArrayD',
1666 doc='Absolute throughput (relative to transmission curves)',
1667 size=pars['COMPABSTHROUGHPUT'].size)
1668 parSchema.addField('compRefOffset', type='ArrayD',
1669 doc='Offset between reference stars and calibrated stars',
1670 size=pars['COMPREFOFFSET'].size)
1671 parSchema.addField('compRefSigma', type='ArrayD',
1672 doc='Width of reference star/calibrated star distribution',
1673 size=pars['COMPREFSIGMA'].size)
1674 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1675 doc='Computed mirror chromaticity terms',
1676 size=pars['COMPMIRRORCHROMATICITY'].size)
1677 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1678 doc='Mirror chromaticity pivot mjd',
1679 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1680 parSchema.addField('compCcdChromaticity', type='ArrayD',
1681 doc='Computed CCD chromaticity terms',
1682 size=pars['COMPCCDCHROMATICITY'].size)
1683 parSchema.addField('compMedianSedSlope', type='ArrayD',
1684 doc='Computed median SED slope (per band)',
1685 size=pars['COMPMEDIANSEDSLOPE'].size)
1686 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1687 size=pars['COMPAPERCORRPIVOT'].size)
1688 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1689 size=pars['COMPAPERCORRSLOPE'].size)
1690 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1691 size=pars['COMPAPERCORRSLOPEERR'].size)
1692 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1693 size=pars['COMPAPERCORRRANGE'].size)
1694 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1695 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1696 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1697 size=pars['COMPMODELERRFWHMPIVOT'].size)
1698 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1699 size=pars['COMPMODELERRSKYPIVOT'].size)
1700 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1701 size=pars['COMPMODELERRPARS'].size)
1702 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1703 size=pars['COMPEXPGRAY'].size)
1704 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1705 size=pars['COMPVARGRAY'].size)
1706 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1707 doc='Computed exposure offset due to background',
1708 size=pars['COMPEXPDELTAMAGBKG'].size)
1709 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1710 doc='Computed number of good stars per exposure',
1711 size=pars['COMPNGOODSTARPEREXP'].size)
1712 parSchema.addField('compExpRefOffset', type='ArrayD',
1713 doc='Computed per-visit median offset between standard stars and ref stars.',
1714 size=pars['COMPEXPREFOFFSET'].size)
1715 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1716 size=pars['COMPSIGFGCM'].size)
1717 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1718 size=pars['COMPSIGMACAL'].size)
1719 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1720 size=pars['COMPRETRIEVEDLNPWV'].size)
1721 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1722 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1723 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1724 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1725 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1726 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1727 parSchema.addField('compEpsilon', type='ArrayD',
1728 doc='Computed epsilon background offset per visit (nJy/arcsec2)',
1729 size=pars['COMPEPSILON'].size)
1730 parSchema.addField('compMedDeltaAper', type='ArrayD',
1731 doc='Median delta mag aper per visit',
1732 size=pars['COMPMEDDELTAAPER'].size)
1733 parSchema.addField('compGlobalEpsilon', type='ArrayD',
1734 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)',
1735 size=pars['COMPGLOBALEPSILON'].size)
1736 parSchema.addField('compEpsilonMap', type='ArrayD',
1737 doc='Computed epsilon maps (nJy/arcsec2)',
1738 size=pars['COMPEPSILONMAP'].size)
1739 parSchema.addField('compEpsilonNStarMap', type='ArrayI',
1740 doc='Number of stars per pixel in computed epsilon maps',
1741 size=pars['COMPEPSILONNSTARMAP'].size)
1742 parSchema.addField('compEpsilonCcdMap', type='ArrayD',
1743 doc='Computed epsilon ccd maps (nJy/arcsec2)',
1744 size=pars['COMPEPSILONCCDMAP'].size)
1745 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI',
1746 doc='Number of stars per ccd bin in epsilon ccd maps',
1747 size=pars['COMPEPSILONCCDNSTARMAP'].size)
1748 # superstarflat section
1749 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1750 size=4)
1751 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1752 size=parSuperStarFlat.size)
1754 return parSchema
1756 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1757 lutFilterNameString, fitBandString):
1758 """
1759 Make the FGCM parameter catalog for persistence
1761 Parameters
1762 ----------
1763 parSchema: `lsst.afw.table.Schema`
1764 Parameter catalog schema
1765 pars: `numpy.ndarray`
1766 FGCM parameters to put into parCat
1767 parSuperStarFlat: `numpy.array`
1768 FGCM superstar flat array to put into parCat
1769 lutFilterNameString: `str`
1770 Combined string of all the lutFilterNames
1771 fitBandString: `str`
1772 Combined string of all the fitBands
1774 Returns
1775 -------
1776 parCat: `afwTable.BasicCatalog`
1777 Atmosphere and instrumental model parameter catalog for persistence
1778 """
1780 parCat = afwTable.BaseCatalog(parSchema)
1781 parCat.reserve(1)
1783 # The parameter catalog just has one row, with many columns for all the
1784 # atmosphere and instrument fit parameters
1785 rec = parCat.addNew()
1787 # info section
1788 rec['nCcd'] = parInfo['NCCD'][0]
1789 rec['lutFilterNames'] = lutFilterNameString
1790 rec['fitBands'] = fitBandString
1791 # note these are not currently supported here.
1792 rec['hasExternalPwv'] = 0
1793 rec['hasExternalTau'] = 0
1795 # parameter section
1797 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1799 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1800 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1801 'parQeSysIntercept', 'compQeSysSlope',
1802 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1803 'parFilterOffset', 'parFilterOffsetFitFlag',
1804 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1805 'compMirrorChromaticity', 'mirrorChromaticityPivot', 'compCcdChromaticity',
1806 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1807 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1808 'compModelErrSkyPivot', 'compModelErrPars',
1809 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1810 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1811 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1812 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper',
1813 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap',
1814 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap', 'compExpRefOffset']
1816 for scalarName in scalarNames:
1817 rec[scalarName] = pars[scalarName.upper()][0]
1819 for arrName in arrNames:
1820 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1822 # superstar section
1823 rec['superstarSize'][:] = parSuperStarFlat.shape
1824 rec['superstar'][:] = parSuperStarFlat.ravel()
1826 return parCat
1828 def _makeFlagStarSchema(self):
1829 """
1830 Make the flagged-stars schema
1832 Returns
1833 -------
1834 flagStarSchema: `lsst.afw.table.Schema`
1835 """
1837 flagStarSchema = afwTable.Schema()
1839 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1840 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1842 return flagStarSchema
1844 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1845 """
1846 Make the flagged star catalog for persistence
1848 Parameters
1849 ----------
1850 flagStarSchema: `lsst.afw.table.Schema`
1851 Flagged star schema
1852 flagStarStruct: `numpy.ndarray`
1853 Flagged star structure from fgcm
1855 Returns
1856 -------
1857 flagStarCat: `lsst.afw.table.BaseCatalog`
1858 Flagged star catalog for persistence
1859 """
1861 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1862 flagStarCat.resize(flagStarStruct.size)
1864 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1865 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1867 return flagStarCat