Coverage for python/lsst/fgcmcal/fgcmFitCycle.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
36import sys
37import traceback
38import copy
40import numpy as np
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45import lsst.afw.table as afwTable
47from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
48from .utilities import extractReferenceMags
49from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
50from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51from .sedterms import SedboundarytermDict, SedtermDict
52from .utilities import lookupStaticCalibrations
54import fgcm
56__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
58MULTIPLE_CYCLES_MAX = 10
61class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
62 dimensions=("instrument",),
63 defaultTemplates={"previousCycleNumber": "-1",
64 "cycleNumber": "0"}):
65 camera = connectionTypes.PrerequisiteInput(
66 doc="Camera instrument",
67 name="camera",
68 storageClass="Camera",
69 dimensions=("instrument",),
70 lookupFunction=lookupStaticCalibrations,
71 isCalibration=True,
72 )
74 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
75 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
76 "chromatic corrections."),
77 name="fgcmLookUpTable",
78 storageClass="Catalog",
79 dimensions=("instrument",),
80 deferLoad=True,
81 )
83 fgcmVisitCatalog = connectionTypes.Input(
84 doc="Catalog of visit information for fgcm",
85 name="fgcmVisitCatalog",
86 storageClass="Catalog",
87 dimensions=("instrument",),
88 deferLoad=True,
89 )
91 fgcmStarObservations = connectionTypes.Input(
92 doc="Catalog of star observations for fgcm",
93 name="fgcmStarObservations",
94 storageClass="Catalog",
95 dimensions=("instrument",),
96 deferLoad=True,
97 )
99 fgcmStarIds = connectionTypes.Input(
100 doc="Catalog of fgcm calibration star IDs",
101 name="fgcmStarIds",
102 storageClass="Catalog",
103 dimensions=("instrument",),
104 deferLoad=True,
105 )
107 fgcmStarIndices = connectionTypes.Input(
108 doc="Catalog of fgcm calibration star indices",
109 name="fgcmStarIndices",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 deferLoad=True,
113 )
115 fgcmReferenceStars = connectionTypes.Input(
116 doc="Catalog of fgcm-matched reference stars",
117 name="fgcmReferenceStars",
118 storageClass="Catalog",
119 dimensions=("instrument",),
120 deferLoad=True,
121 )
123 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
124 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
125 name="fgcmFlaggedStars{previousCycleNumber}",
126 storageClass="Catalog",
127 dimensions=("instrument",),
128 deferLoad=True,
129 )
131 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
132 doc="Catalog of fgcm fit parameters from previous fit cycle",
133 name="fgcmFitParameters{previousCycleNumber}",
134 storageClass="Catalog",
135 dimensions=("instrument",),
136 deferLoad=True,
137 )
139 fgcmFitParameters = connectionTypes.Output(
140 doc="Catalog of fgcm fit parameters from current fit cycle",
141 name="fgcmFitParameters{cycleNumber}",
142 storageClass="Catalog",
143 dimensions=("instrument",),
144 )
146 fgcmFlaggedStars = connectionTypes.Output(
147 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
148 name="fgcmFlaggedStars{cycleNumber}",
149 storageClass="Catalog",
150 dimensions=("instrument",),
151 )
153 fgcmZeropoints = connectionTypes.Output(
154 doc="Catalog of fgcm zeropoint data from current fit cycle",
155 name="fgcmZeropoints{cycleNumber}",
156 storageClass="Catalog",
157 dimensions=("instrument",),
158 )
160 fgcmAtmosphereParameters = connectionTypes.Output(
161 doc="Catalog of atmospheric fit parameters from current fit cycle",
162 name="fgcmAtmosphereParameters{cycleNumber}",
163 storageClass="Catalog",
164 dimensions=("instrument",),
165 )
167 fgcmStandardStars = connectionTypes.Output(
168 doc="Catalog of standard star magnitudes from current fit cycle",
169 name="fgcmStandardStars{cycleNumber}",
170 storageClass="SimpleCatalog",
171 dimensions=("instrument",),
172 )
174 # Add connections for running multiple cycles
175 # This uses vars() to alter the class __dict__ to programmatically
176 # write many similar outputs.
177 for cycle in range(MULTIPLE_CYCLES_MAX):
178 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
179 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
180 name=f"fgcmFitParameters{cycle}",
181 storageClass="Catalog",
182 dimensions=("instrument",),
183 )
184 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
185 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
186 name=f"fgcmFlaggedStars{cycle}",
187 storageClass="Catalog",
188 dimensions=("instrument",),
189 )
190 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
191 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
192 name=f"fgcmZeropoints{cycle}",
193 storageClass="Catalog",
194 dimensions=("instrument",),
195 )
196 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
197 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
198 name=f"fgcmAtmosphereParameters{cycle}",
199 storageClass="Catalog",
200 dimensions=("instrument",),
201 )
202 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
203 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
204 name=f"fgcmStandardStars{cycle}",
205 storageClass="SimpleCatalog",
206 dimensions=("instrument",),
207 )
209 def __init__(self, *, config=None):
210 super().__init__(config=config)
212 if not config.doReferenceCalibration:
213 self.inputs.remove("fgcmReferenceStars")
215 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
216 raise ValueError("cycleNumber must be of integer format")
217 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
218 raise ValueError("previousCycleNumber must be of integer format")
219 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
220 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
222 if int(config.connections.cycleNumber) == 0:
223 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
224 self.prerequisiteInputs.remove("fgcmFitParametersInput")
226 if not self.config.doMultipleCycles:
227 # Single-cycle run
228 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
229 self.outputs.remove("fgcmStandardStars")
231 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
232 self.outputs.remove("fgcmZeropoints")
233 self.outputs.remove("fgcmAtmosphereParameters")
235 # Remove all multiple cycle outputs
236 for cycle in range(0, MULTIPLE_CYCLES_MAX):
237 self.outputs.remove(f"fgcmFitParameters{cycle}")
238 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
239 self.outputs.remove(f"fgcmZeropoints{cycle}")
240 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
241 self.outputs.remove(f"fgcmStandardStars{cycle}")
243 else:
244 # Multiple-cycle run
245 # Remove single-cycle outputs
246 self.outputs.remove("fgcmFitParameters")
247 self.outputs.remove("fgcmFlaggedStars")
248 self.outputs.remove("fgcmZeropoints")
249 self.outputs.remove("fgcmAtmosphereParameters")
250 self.outputs.remove("fgcmStandardStars")
252 # Remove outputs from cycles that are not used
253 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
254 MULTIPLE_CYCLES_MAX):
255 self.outputs.remove(f"fgcmFitParameters{cycle}")
256 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
257 self.outputs.remove(f"fgcmZeropoints{cycle}")
258 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
259 self.outputs.remove(f"fgcmStandardStars{cycle}")
261 # Remove non-final-cycle outputs if necessary
262 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
263 if not self.config.outputZeropointsBeforeFinalCycle:
264 self.outputs.remove(f"fgcmZeropoints{cycle}")
265 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
266 if not self.config.outputStandardsBeforeFinalCycle:
267 self.outputs.remove(f"fgcmStandardStars{cycle}")
270class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
271 pipelineConnections=FgcmFitCycleConnections):
272 """Config for FgcmFitCycle"""
274 doMultipleCycles = pexConfig.Field(
275 doc="Run multiple fit cycles in one task",
276 dtype=bool,
277 default=False,
278 )
279 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
280 doc=("Final cycle number in multiple cycle mode. The initial cycle "
281 "is 0, with limited parameters fit. The next cycle is 1 with "
282 "full parameter fit. The final cycle is a clean-up with no "
283 "parameters fit. There will be a total of "
284 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
285 "cycle number cannot be less than 2."),
286 dtype=int,
287 default=5,
288 min=2,
289 max=MULTIPLE_CYCLES_MAX,
290 inclusiveMax=True,
291 )
292 bands = pexConfig.ListField(
293 doc="Bands to run calibration",
294 dtype=str,
295 default=[],
296 )
297 fitFlag = pexConfig.ListField(
298 doc=("Flag for which bands are directly constrained in the FGCM fit. "
299 "Bands set to 0 will have the atmosphere constrained from observations "
300 "in other bands on the same night. Must be same length as config.bands, "
301 "and matched band-by-band."),
302 dtype=int,
303 default=(0,),
304 optional=True,
305 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
306 "It will be removed after v20. Use fitBands instead."),
307 )
308 fitBands = pexConfig.ListField(
309 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
310 "the atmosphere constrained from the 'fitBands' on the same night. "
311 "Must be a subset of `config.bands`"),
312 dtype=str,
313 default=[],
314 )
315 requiredFlag = pexConfig.ListField(
316 doc=("Flag for which bands are required for a star to be considered a calibration "
317 "star in the FGCM fit. Typically this should be the same as fitFlag. Must "
318 "be same length as config.bands, and matched band-by-band."),
319 dtype=int,
320 default=(0,),
321 optional=True,
322 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
323 "It will be removed after v20. Use requiredBands instead."),
324 )
325 requiredBands = pexConfig.ListField(
326 doc=("Bands that are required for a star to be considered a calibration star. "
327 "Must be a subset of `config.bands`"),
328 dtype=str,
329 default=[],
330 )
331 filterMap = pexConfig.DictField(
332 doc="Mapping from 'filterName' to band.",
333 keytype=str,
334 itemtype=str,
335 default={},
336 deprecated=("This field is no longer used, and has been deprecated by "
337 "DM-28088. It will be removed after v22. Use "
338 "physicalFilterMap instead.")
339 )
340 # The following config will not be necessary after Gen2 retirement.
341 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
342 # is easiest to access in the config file.
343 physicalFilterMap = pexConfig.DictField(
344 doc="Mapping from 'physicalFilter' to band.",
345 keytype=str,
346 itemtype=str,
347 default={},
348 )
349 doReferenceCalibration = pexConfig.Field(
350 doc="Use reference catalog as additional constraint on calibration",
351 dtype=bool,
352 default=True,
353 )
354 refStarSnMin = pexConfig.Field(
355 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
356 dtype=float,
357 default=50.0,
358 )
359 refStarOutlierNSig = pexConfig.Field(
360 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
361 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
362 dtype=float,
363 default=4.0,
364 )
365 applyRefStarColorCuts = pexConfig.Field(
366 doc="Apply color cuts to reference stars?",
367 dtype=bool,
368 default=True,
369 )
370 nCore = pexConfig.Field(
371 doc="Number of cores to use",
372 dtype=int,
373 default=4,
374 )
375 nStarPerRun = pexConfig.Field(
376 doc="Number of stars to run in each chunk",
377 dtype=int,
378 default=200000,
379 )
380 nExpPerRun = pexConfig.Field(
381 doc="Number of exposures to run in each chunk",
382 dtype=int,
383 default=1000,
384 )
385 reserveFraction = pexConfig.Field(
386 doc="Fraction of stars to reserve for testing",
387 dtype=float,
388 default=0.1,
389 )
390 freezeStdAtmosphere = pexConfig.Field(
391 doc="Freeze atmosphere parameters to standard (for testing)",
392 dtype=bool,
393 default=False,
394 )
395 precomputeSuperStarInitialCycle = pexConfig.Field(
396 doc="Precompute superstar flat for initial cycle",
397 dtype=bool,
398 default=False,
399 )
400 superStarSubCcd = pexConfig.Field(
401 doc="Compute superstar flat on sub-ccd scale",
402 dtype=bool,
403 default=True,
404 optional=True,
405 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
406 "It will be removed after v20. Use superStarSubCcdDict instead."),
407 )
408 superStarSubCcdDict = pexConfig.DictField(
409 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
410 "Must have one entry per band."),
411 keytype=str,
412 itemtype=bool,
413 default={},
414 )
415 superStarSubCcdChebyshevOrder = pexConfig.Field(
416 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
417 "Global default is first-order polynomials, and should be overridden "
418 "on a camera-by-camera basis depending on the ISR."),
419 dtype=int,
420 default=1,
421 )
422 superStarSubCcdTriangular = pexConfig.Field(
423 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
424 "suppress high-order cross terms?"),
425 dtype=bool,
426 default=False,
427 )
428 superStarSigmaClip = pexConfig.Field(
429 doc="Number of sigma to clip outliers when selecting for superstar flats",
430 dtype=float,
431 default=5.0,
432 )
433 focalPlaneSigmaClip = pexConfig.Field(
434 doc="Number of sigma to clip outliers per focal-plane.",
435 dtype=float,
436 default=4.0,
437 )
438 ccdGraySubCcd = pexConfig.Field(
439 doc="Compute CCD gray terms on sub-ccd scale",
440 dtype=bool,
441 default=False,
442 optional=True,
443 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
444 "It will be removed after v20. Use ccdGraySubCcdDict instead."),
445 )
446 ccdGraySubCcdDict = pexConfig.DictField(
447 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
448 "('ccd gray') on a sub-ccd scale."),
449 keytype=str,
450 itemtype=bool,
451 default={},
452 )
453 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
454 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
455 dtype=int,
456 default=1,
457 )
458 ccdGraySubCcdTriangular = pexConfig.Field(
459 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
460 "suppress high-order cross terms?"),
461 dtype=bool,
462 default=True,
463 )
464 ccdGrayFocalPlaneDict = pexConfig.DictField(
465 doc=("Per-band specification on whether to compute focal-plane residual "
466 "('ccd gray') corrections."),
467 keytype=str,
468 itemtype=bool,
469 default={},
470 )
471 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
472 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
473 "gray corrections. If there are fewer good CCDs then the gray "
474 "correction is computed per-ccd."),
475 dtype=int,
476 default=1,
477 )
478 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
479 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
480 dtype=int,
481 default=3,
482 )
483 cycleNumber = pexConfig.Field(
484 doc=("FGCM fit cycle number. This is automatically incremented after each run "
485 "and stage of outlier rejection. See cookbook for details."),
486 dtype=int,
487 default=None,
488 )
489 isFinalCycle = pexConfig.Field(
490 doc=("Is this the final cycle of the fitting? Will automatically compute final "
491 "selection of stars and photometric exposures, and will output zeropoints "
492 "and standard stars for use in fgcmOutputProducts"),
493 dtype=bool,
494 default=False,
495 )
496 maxIterBeforeFinalCycle = pexConfig.Field(
497 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
498 "will always be 0 in the final cycle for cleanup and final selection."),
499 dtype=int,
500 default=50,
501 )
502 deltaMagBkgOffsetPercentile = pexConfig.Field(
503 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
504 "offset from local background subtraction."),
505 dtype=float,
506 default=0.25,
507 )
508 deltaMagBkgPerCcd = pexConfig.Field(
509 doc=("Compute net offset from local background subtraction per-ccd? "
510 "Otherwise, use computation per visit."),
511 dtype=bool,
512 default=False,
513 )
514 utBoundary = pexConfig.Field(
515 doc="Boundary (in UTC) from day-to-day",
516 dtype=float,
517 default=None,
518 )
519 washMjds = pexConfig.ListField(
520 doc="Mirror wash MJDs",
521 dtype=float,
522 default=(0.0,),
523 )
524 epochMjds = pexConfig.ListField(
525 doc="Epoch boundaries in MJD",
526 dtype=float,
527 default=(0.0,),
528 )
529 minObsPerBand = pexConfig.Field(
530 doc="Minimum good observations per band",
531 dtype=int,
532 default=2,
533 )
534 # TODO: When DM-16511 is done, it will be possible to get the
535 # telescope latitude directly from the camera.
536 latitude = pexConfig.Field(
537 doc="Observatory latitude",
538 dtype=float,
539 default=None,
540 )
541 brightObsGrayMax = pexConfig.Field(
542 doc="Maximum gray extinction to be considered bright observation",
543 dtype=float,
544 default=0.15,
545 )
546 minStarPerCcd = pexConfig.Field(
547 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
548 "CCDs with fewer stars will have their calibration estimated from other "
549 "CCDs in the same visit, with zeropoint error increased accordingly."),
550 dtype=int,
551 default=5,
552 )
553 minCcdPerExp = pexConfig.Field(
554 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
555 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
556 dtype=int,
557 default=5,
558 )
559 maxCcdGrayErr = pexConfig.Field(
560 doc="Maximum error on CCD gray offset to be considered photometric",
561 dtype=float,
562 default=0.05,
563 )
564 minStarPerExp = pexConfig.Field(
565 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
566 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
567 dtype=int,
568 default=600,
569 )
570 minExpPerNight = pexConfig.Field(
571 doc="Minimum number of good exposures/visits to consider a partly photometric night",
572 dtype=int,
573 default=10,
574 )
575 expGrayInitialCut = pexConfig.Field(
576 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
577 "observations."),
578 dtype=float,
579 default=-0.25,
580 )
581 expGrayPhotometricCut = pexConfig.ListField(
582 doc=("Maximum (negative) exposure gray for a visit to be considered photometric. "
583 "Must be same length as config.bands, and matched band-by-band."),
584 dtype=float,
585 default=(0.0,),
586 optional=True,
587 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
588 "It will be removed after v20. Use expGrayPhotometricCutDict instead."),
589 )
590 expGrayPhotometricCutDict = pexConfig.DictField(
591 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
592 "('gray term') for a visit to be considered photometric. Must have one "
593 "entry per band. Broad-band filters should be -0.05."),
594 keytype=str,
595 itemtype=float,
596 default={},
597 )
598 expGrayHighCut = pexConfig.ListField(
599 doc=("Maximum (positive) exposure gray for a visit to be considered photometric. "
600 "Must be same length as config.bands, and matched band-by-band."),
601 dtype=float,
602 default=(0.0,),
603 optional=True,
604 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
605 "It will be removed after v20. Use expGrayHighCutDict instead."),
606 )
607 expGrayHighCutDict = pexConfig.DictField(
608 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
609 "('gray term') for a visit to be considered photometric. Must have one "
610 "entry per band. Broad-band filters should be 0.2."),
611 keytype=str,
612 itemtype=float,
613 default={},
614 )
615 expGrayRecoverCut = pexConfig.Field(
616 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
617 "Visits with more gray extinction will only get CCD zeropoints if there are "
618 "sufficient star observations (minStarPerCcd) on that CCD."),
619 dtype=float,
620 default=-1.0,
621 )
622 expVarGrayPhotometricCut = pexConfig.Field(
623 doc="Maximum exposure variance to be considered possibly photometric",
624 dtype=float,
625 default=0.0005,
626 optional=True,
627 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
628 "It will be removed after v20. Use expVarGrayPhotometricCutDict instead."),
629 )
630 expVarGrayPhotometricCutDict = pexConfig.DictField(
631 doc=("Per-band specification on maximum exposure variance to be considered possibly "
632 "photometric. Must have one entry per band. Broad-band filters should be "
633 "0.0005."),
634 keytype=str,
635 itemtype=float,
636 default={},
637 )
638 expGrayErrRecoverCut = pexConfig.Field(
639 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
640 "Visits with more gray variance will only get CCD zeropoints if there are "
641 "sufficient star observations (minStarPerCcd) on that CCD."),
642 dtype=float,
643 default=0.05,
644 )
645 aperCorrFitNBins = pexConfig.Field(
646 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
647 "no fit will be performed, and the config.aperCorrInputSlopes will be "
648 "used if available."),
649 dtype=int,
650 default=10,
651 )
652 aperCorrInputSlopes = pexConfig.ListField(
653 doc=("Aperture correction input slope parameters. These are used on the first "
654 "fit iteration, and aperture correction parameters will be updated from "
655 "the data if config.aperCorrFitNBins > 0. It is recommended to set this"
656 "when there is insufficient data to fit the parameters (e.g. tract mode). "
657 "If set, must be same length as config.bands, and matched band-by-band."),
658 dtype=float,
659 default=[],
660 optional=True,
661 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
662 "It will be removed after v20. Use aperCorrInputSlopeDict instead."),
663 )
664 aperCorrInputSlopeDict = pexConfig.DictField(
665 doc=("Per-band specification of aperture correction input slope parameters. These "
666 "are used on the first fit iteration, and aperture correction parameters will "
667 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
668 "to set this when there is insufficient data to fit the parameters (e.g. "
669 "tract mode)."),
670 keytype=str,
671 itemtype=float,
672 default={},
673 )
674 sedFudgeFactors = pexConfig.ListField(
675 doc=("Fudge factors for computing linear SED from colors. Must be same length as "
676 "config.bands, and matched band-by-band."),
677 dtype=float,
678 default=(0,),
679 optional=True,
680 deprecated=("This field has been deprecated and will be removed after v20. "
681 "Please use sedSlopeTermMap and sedSlopeMap."),
682 )
683 sedboundaryterms = pexConfig.ConfigField(
684 doc="Mapping from bands to SED boundary term names used is sedterms.",
685 dtype=SedboundarytermDict,
686 )
687 sedterms = pexConfig.ConfigField(
688 doc="Mapping from terms to bands for fgcm linear SED approximations.",
689 dtype=SedtermDict,
690 )
691 sigFgcmMaxErr = pexConfig.Field(
692 doc="Maximum mag error for fitting sigma_FGCM",
693 dtype=float,
694 default=0.01,
695 )
696 sigFgcmMaxEGray = pexConfig.ListField(
697 doc=("Maximum (absolute) gray value for observation in sigma_FGCM. "
698 "May be 1 element (same for all bands) or the same length as config.bands."),
699 dtype=float,
700 default=(0.05,),
701 optional=True,
702 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
703 "It will be removed after v20. Use sigFgcmMaxEGrayDict instead."),
704 )
705 sigFgcmMaxEGrayDict = pexConfig.DictField(
706 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
707 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
708 "should be 0.05."),
709 keytype=str,
710 itemtype=float,
711 default={},
712 )
713 ccdGrayMaxStarErr = pexConfig.Field(
714 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
715 "computation"),
716 dtype=float,
717 default=0.10,
718 )
719 approxThroughput = pexConfig.ListField(
720 doc=("Approximate overall throughput at start of calibration observations. "
721 "May be 1 element (same for all bands) or the same length as config.bands."),
722 dtype=float,
723 default=(1.0, ),
724 optional=True,
725 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
726 "It will be removed after v20. Use approxThroughputDict instead."),
727 )
728 approxThroughputDict = pexConfig.DictField(
729 doc=("Per-band specification of the approximate overall throughput at the start of "
730 "calibration observations. Must have one entry per band. Typically should "
731 "be 1.0."),
732 keytype=str,
733 itemtype=float,
734 default={},
735 )
736 sigmaCalRange = pexConfig.ListField(
737 doc="Allowed range for systematic error floor estimation",
738 dtype=float,
739 default=(0.001, 0.003),
740 )
741 sigmaCalFitPercentile = pexConfig.ListField(
742 doc="Magnitude percentile range to fit systematic error floor",
743 dtype=float,
744 default=(0.05, 0.15),
745 )
746 sigmaCalPlotPercentile = pexConfig.ListField(
747 doc="Magnitude percentile range to plot systematic error floor",
748 dtype=float,
749 default=(0.05, 0.95),
750 )
751 sigma0Phot = pexConfig.Field(
752 doc="Systematic error floor for all zeropoints",
753 dtype=float,
754 default=0.003,
755 )
756 mapLongitudeRef = pexConfig.Field(
757 doc="Reference longitude for plotting maps",
758 dtype=float,
759 default=0.0,
760 )
761 mapNSide = pexConfig.Field(
762 doc="Healpix nside for plotting maps",
763 dtype=int,
764 default=256,
765 )
766 outfileBase = pexConfig.Field(
767 doc="Filename start for plot output files",
768 dtype=str,
769 default=None,
770 )
771 starColorCuts = pexConfig.ListField(
772 doc="Encoded star-color cuts (to be cleaned up)",
773 dtype=str,
774 default=("NO_DATA",),
775 )
776 colorSplitIndices = pexConfig.ListField(
777 doc="Band indices to use to split stars by color",
778 dtype=int,
779 default=None,
780 optional=True,
781 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
782 "It will be removed after v20. Use colorSplitBands instead."),
783 )
784 colorSplitBands = pexConfig.ListField(
785 doc="Band names to use to split stars by color. Must have 2 entries.",
786 dtype=str,
787 length=2,
788 default=('g', 'i'),
789 )
790 modelMagErrors = pexConfig.Field(
791 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
792 dtype=bool,
793 default=True,
794 )
795 useQuadraticPwv = pexConfig.Field(
796 doc="Model PWV with a quadratic term for variation through the night?",
797 dtype=bool,
798 default=False,
799 )
800 instrumentParsPerBand = pexConfig.Field(
801 doc=("Model instrumental parameters per band? "
802 "Otherwise, instrumental parameters (QE changes with time) are "
803 "shared among all bands."),
804 dtype=bool,
805 default=False,
806 )
807 instrumentSlopeMinDeltaT = pexConfig.Field(
808 doc=("Minimum time change (in days) between observations to use in constraining "
809 "instrument slope."),
810 dtype=float,
811 default=20.0,
812 )
813 fitMirrorChromaticity = pexConfig.Field(
814 doc="Fit (intraband) mirror chromatic term?",
815 dtype=bool,
816 default=False,
817 )
818 coatingMjds = pexConfig.ListField(
819 doc="Mirror coating dates in MJD",
820 dtype=float,
821 default=(0.0,),
822 )
823 outputStandardsBeforeFinalCycle = pexConfig.Field(
824 doc="Output standard stars prior to final cycle? Used in debugging.",
825 dtype=bool,
826 default=False,
827 )
828 outputZeropointsBeforeFinalCycle = pexConfig.Field(
829 doc="Output standard stars prior to final cycle? Used in debugging.",
830 dtype=bool,
831 default=False,
832 )
833 useRepeatabilityForExpGrayCuts = pexConfig.ListField(
834 doc=("Use star repeatability (instead of exposures) for computing photometric "
835 "cuts? Recommended for tract mode or bands with few exposures. "
836 "May be 1 element (same for all bands) or the same length as config.bands."),
837 dtype=bool,
838 default=(False,),
839 optional=True,
840 deprecated=("This field is no longer used, and has been deprecated by DM-23699. "
841 "It will be removed after v20. Use useRepeatabilityForExpGrayCutsDict instead."),
842 )
843 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
844 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
845 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
846 keytype=str,
847 itemtype=bool,
848 default={},
849 )
850 autoPhotometricCutNSig = pexConfig.Field(
851 doc=("Number of sigma for automatic computation of (low) photometric cut. "
852 "Cut is based on exposure gray width (per band), unless "
853 "useRepeatabilityForExpGrayCuts is set, in which case the star "
854 "repeatability is used (also per band)."),
855 dtype=float,
856 default=3.0,
857 )
858 autoHighCutNSig = pexConfig.Field(
859 doc=("Number of sigma for automatic computation of (high) outlier cut. "
860 "Cut is based on exposure gray width (per band), unless "
861 "useRepeatabilityForExpGrayCuts is set, in which case the star "
862 "repeatability is used (also per band)."),
863 dtype=float,
864 default=4.0,
865 )
866 quietMode = pexConfig.Field(
867 doc="Be less verbose with logging.",
868 dtype=bool,
869 default=False,
870 )
871 doPlots = pexConfig.Field(
872 doc="Make fgcm QA plots.",
873 dtype=bool,
874 default=True,
875 )
876 randomSeed = pexConfig.Field(
877 doc="Random seed for fgcm for consistency in tests.",
878 dtype=int,
879 default=None,
880 optional=True,
881 )
883 def validate(self):
884 super().validate()
886 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
887 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
888 raise RuntimeError(msg)
889 if self.connections.cycleNumber != str(self.cycleNumber):
890 msg = "cycleNumber in template must be equal to connections.cycleNumber"
891 raise RuntimeError(msg)
893 for band in self.fitBands:
894 if band not in self.bands:
895 msg = 'fitBand %s not in bands' % (band)
896 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
897 for band in self.requiredBands:
898 if band not in self.bands:
899 msg = 'requiredBand %s not in bands' % (band)
900 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
901 for band in self.colorSplitBands:
902 if band not in self.bands:
903 msg = 'colorSplitBand %s not in bands' % (band)
904 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
905 for band in self.bands:
906 if band not in self.superStarSubCcdDict:
907 msg = 'band %s not in superStarSubCcdDict' % (band)
908 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
909 self, msg)
910 if band not in self.ccdGraySubCcdDict:
911 msg = 'band %s not in ccdGraySubCcdDict' % (band)
912 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
913 self, msg)
914 if band not in self.expGrayPhotometricCutDict:
915 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
916 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
917 self, msg)
918 if band not in self.expGrayHighCutDict:
919 msg = 'band %s not in expGrayHighCutDict' % (band)
920 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
921 self, msg)
922 if band not in self.expVarGrayPhotometricCutDict:
923 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
924 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
925 self, msg)
926 if band not in self.sigFgcmMaxEGrayDict:
927 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
928 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
929 self, msg)
930 if band not in self.approxThroughputDict:
931 msg = 'band %s not in approxThroughputDict' % (band)
932 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
933 self, msg)
934 if band not in self.useRepeatabilityForExpGrayCutsDict:
935 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
936 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
937 self, msg)
940class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
941 """Subclass of TaskRunner for fgcmFitCycleTask
943 fgcmFitCycleTask.run() takes one argument, the butler, and uses
944 stars and visits previously extracted from dataRefs by
945 fgcmBuildStars.
946 This Runner does not perform any dataRef parallelization, but the FGCM
947 code called by the Task uses python multiprocessing (see the "ncores"
948 config option).
949 """
951 @staticmethod
952 def getTargetList(parsedCmd):
953 """
954 Return a list with one element, the butler.
955 """
956 return [parsedCmd.butler]
958 def __call__(self, butler):
959 """
960 Parameters
961 ----------
962 butler: `lsst.daf.persistence.Butler`
964 Returns
965 -------
966 exitStatus: `list` with `pipeBase.Struct`
967 exitStatus (0: success; 1: failure)
968 """
970 task = self.TaskClass(config=self.config, log=self.log)
972 exitStatus = 0
973 if self.doRaise:
974 task.runDataRef(butler)
975 else:
976 try:
977 task.runDataRef(butler)
978 except Exception as e:
979 exitStatus = 1
980 task.log.fatal("Failed: %s" % e)
981 if not isinstance(e, pipeBase.TaskError):
982 traceback.print_exc(file=sys.stderr)
984 task.writeMetadata(butler)
986 # The task does not return any results:
987 return [pipeBase.Struct(exitStatus=exitStatus)]
989 def run(self, parsedCmd):
990 """
991 Run the task, with no multiprocessing
993 Parameters
994 ----------
995 parsedCmd: ArgumentParser parsed command line
996 """
998 resultList = []
1000 if self.precall(parsedCmd):
1001 targetList = self.getTargetList(parsedCmd)
1002 # make sure that we only get 1
1003 resultList = self(targetList[0])
1005 return resultList
1008class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
1009 """
1010 Run Single fit cycle for FGCM global calibration
1011 """
1013 ConfigClass = FgcmFitCycleConfig
1014 RunnerClass = FgcmFitCycleRunner
1015 _DefaultName = "fgcmFitCycle"
1017 def __init__(self, butler=None, initInputs=None, **kwargs):
1018 super().__init__(**kwargs)
1020 # no saving of metadata for now
1021 def _getMetadataName(self):
1022 return None
1024 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1025 camera = butlerQC.get(inputRefs.camera)
1027 dataRefDict = {}
1029 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
1030 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
1031 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
1032 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
1033 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
1034 if self.config.doReferenceCalibration:
1035 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
1036 if self.config.cycleNumber > 0:
1037 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
1038 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
1040 fgcmDatasetDict = None
1041 if self.config.doMultipleCycles:
1042 # Run multiple cycles at once.
1043 config = copy.copy(self.config)
1044 config.update(cycleNumber=0)
1045 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
1046 if cycle == self.config.multipleCyclesFinalCycleNumber:
1047 config.update(isFinalCycle=True)
1049 if cycle > 0:
1050 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
1051 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
1053 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
1054 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
1055 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
1056 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
1057 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
1058 if self.outputZeropoints:
1059 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
1060 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
1061 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
1062 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
1063 if self.outputStandards:
1064 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
1065 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
1066 else:
1067 # Run a single cycle
1068 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
1070 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
1071 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
1072 if self.outputZeropoints:
1073 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
1074 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
1075 if self.outputStandards:
1076 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
1078 @pipeBase.timeMethod
1079 def runDataRef(self, butler):
1080 """
1081 Run a single fit cycle for FGCM
1083 Parameters
1084 ----------
1085 butler: `lsst.daf.persistence.Butler`
1086 """
1087 self._checkDatasetsExist(butler)
1089 dataRefDict = {}
1090 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
1091 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
1092 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
1093 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
1094 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
1095 if self.config.doReferenceCalibration:
1096 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
1097 if self.config.cycleNumber > 0:
1098 lastCycle = self.config.cycleNumber - 1
1099 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
1100 fgcmcycle=lastCycle)
1101 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
1102 fgcmcycle=lastCycle)
1104 camera = butler.get('camera')
1105 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
1107 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
1108 fgcmcycle=self.config.cycleNumber)
1109 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
1110 fgcmcycle=self.config.cycleNumber)
1111 if self.outputZeropoints:
1112 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
1113 fgcmcycle=self.config.cycleNumber)
1114 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
1115 fgcmcycle=self.config.cycleNumber)
1116 if self.outputStandards:
1117 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
1118 fgcmcycle=self.config.cycleNumber)
1120 def writeConfig(self, butler, clobber=False, doBackup=True):
1121 """Write the configuration used for processing the data, or check that an existing
1122 one is equal to the new one if present. This is an override of the regular
1123 version from pipe_base that knows about fgcmcycle.
1125 Parameters
1126 ----------
1127 butler : `lsst.daf.persistence.Butler`
1128 Data butler used to write the config. The config is written to dataset type
1129 `CmdLineTask._getConfigName`.
1130 clobber : `bool`, optional
1131 A boolean flag that controls what happens if a config already has been saved:
1132 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1133 - `False`: raise `TaskError` if this config does not match the existing config.
1134 doBackup : `bool`, optional
1135 Set to `True` to backup the config files if clobbering.
1136 """
1137 configName = self._getConfigName()
1138 if configName is None:
1139 return
1140 if clobber:
1141 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1142 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1143 # this may be subject to a race condition; see #2789
1144 try:
1145 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1146 except Exception as exc:
1147 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1148 (configName, exc))
1150 def logConfigMismatch(msg):
1151 self.log.fatal("Comparing configuration: %s", msg)
1153 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1154 raise pipeBase.TaskError(
1155 f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1156 " must be consistent within the same output repo (override with --clobber-config)")
1157 else:
1158 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1160 def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1161 """
1162 Run the fit cycle
1164 Parameters
1165 ----------
1166 camera : `lsst.afw.cameraGeom.Camera`
1167 dataRefDict : `dict`
1168 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1169 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1170 dataRef dictionary with keys:
1172 ``"fgcmLookUpTable"``
1173 dataRef for the FGCM look-up table.
1174 ``"fgcmVisitCatalog"``
1175 dataRef for visit summary catalog.
1176 ``"fgcmStarObservations"``
1177 dataRef for star observation catalog.
1178 ``"fgcmStarIds"``
1179 dataRef for star id catalog.
1180 ``"fgcmStarIndices"``
1181 dataRef for star index catalog.
1182 ``"fgcmReferenceStars"``
1183 dataRef for matched reference star catalog.
1184 ``"fgcmFlaggedStars"``
1185 dataRef for flagged star catalog.
1186 ``"fgcmFitParameters"``
1187 dataRef for fit parameter catalog.
1188 config : `lsst.pex.config.Config`, optional
1189 Configuration to use to override self.config.
1191 Returns
1192 -------
1193 fgcmDatasetDict : `dict`
1194 Dictionary of datasets to persist.
1195 """
1196 if config is not None:
1197 _config = config
1198 else:
1199 _config = self.config
1201 # Set defaults on whether to output standards and zeropoints
1202 self.maxIter = _config.maxIterBeforeFinalCycle
1203 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1204 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1205 self.resetFitParameters = True
1207 if _config.isFinalCycle:
1208 # This is the final fit cycle, so we do not want to reset fit
1209 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1210 # and we always want to output standards and zeropoints
1211 self.maxIter = 0
1212 self.outputStandards = True
1213 self.outputZeropoints = True
1214 self.resetFitParameters = False
1216 lutCat = dataRefDict['fgcmLookUpTable'].get()
1217 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1218 dict(_config.physicalFilterMap))
1219 del lutCat
1221 configDict = makeConfigDict(_config, self.log, camera,
1222 self.maxIter, self.resetFitParameters,
1223 self.outputZeropoints,
1224 lutIndexVals[0]['FILTERNAMES'])
1226 # next we need the exposure/visit information
1227 visitCat = dataRefDict['fgcmVisitCatalog'].get()
1228 fgcmExpInfo = translateVisitCatalog(visitCat)
1229 del visitCat
1231 # Use the first orientation.
1232 # TODO: DM-21215 will generalize to arbitrary camera orientations
1233 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1235 noFitsDict = {'lutIndex': lutIndexVals,
1236 'lutStd': lutStd,
1237 'expInfo': fgcmExpInfo,
1238 'ccdOffsets': ccdOffsets}
1240 # set up the fitter object
1241 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1242 noFitsDict=noFitsDict, noOutput=True)
1244 # create the parameter object
1245 if (fgcmFitCycle.initialCycle):
1246 # cycle = 0, initial cycle
1247 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1248 fgcmLut,
1249 fgcmExpInfo)
1250 else:
1251 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog):
1252 parCat = dataRefDict['fgcmFitParameters']
1253 else:
1254 parCat = dataRefDict['fgcmFitParameters'].get()
1255 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1256 del parCat
1257 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1258 fgcmExpInfo,
1259 inParInfo,
1260 inParams,
1261 inSuperStar)
1263 # set up the stars...
1264 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1266 starObs = dataRefDict['fgcmStarObservations'].get()
1267 starIds = dataRefDict['fgcmStarIds'].get()
1268 starIndices = dataRefDict['fgcmStarIndices'].get()
1270 # grab the flagged stars if available
1271 if 'fgcmFlaggedStars' in dataRefDict:
1272 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1273 flaggedStars = dataRefDict['fgcmFlaggedStars']
1274 else:
1275 flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1276 flagId = flaggedStars['objId'][:]
1277 flagFlag = flaggedStars['objFlag'][:]
1278 else:
1279 flaggedStars = None
1280 flagId = None
1281 flagFlag = None
1283 if _config.doReferenceCalibration:
1284 refStars = dataRefDict['fgcmReferenceStars'].get()
1286 refMag, refMagErr = extractReferenceMags(refStars,
1287 _config.bands,
1288 _config.physicalFilterMap)
1289 refId = refStars['fgcm_id'][:]
1290 else:
1291 refStars = None
1292 refId = None
1293 refMag = None
1294 refMagErr = None
1296 # match star observations to visits
1297 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1298 # actually be transferred into fgcm using the indexing below.
1299 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1301 # The fgcmStars.loadStars method will copy all the star information into
1302 # special shared memory objects that will not blow up the memory usage when
1303 # used with python multiprocessing. Once all the numbers are copied,
1304 # it is necessary to release all references to the objects that previously
1305 # stored the data to ensure that the garbage collector can clear the memory,
1306 # and ensure that this memory is not copied when multiprocessing kicks in.
1308 # We determine the conversion from the native units (typically radians) to
1309 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1310 # numpy arrays rather than Angles, which would we approximately 600x slower.
1311 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1313 fgcmStars.loadStars(fgcmPars,
1314 starObs['visit'][starIndices['obsIndex']],
1315 starObs['ccd'][starIndices['obsIndex']],
1316 starObs['ra'][starIndices['obsIndex']] * conv,
1317 starObs['dec'][starIndices['obsIndex']] * conv,
1318 starObs['instMag'][starIndices['obsIndex']],
1319 starObs['instMagErr'][starIndices['obsIndex']],
1320 fgcmExpInfo['FILTERNAME'][visitIndex],
1321 starIds['fgcm_id'][:],
1322 starIds['ra'][:],
1323 starIds['dec'][:],
1324 starIds['obsArrIndex'][:],
1325 starIds['nObs'][:],
1326 obsX=starObs['x'][starIndices['obsIndex']],
1327 obsY=starObs['y'][starIndices['obsIndex']],
1328 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1329 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1330 refID=refId,
1331 refMag=refMag,
1332 refMagErr=refMagErr,
1333 flagID=flagId,
1334 flagFlag=flagFlag,
1335 computeNobs=True)
1337 # Release all references to temporary objects holding star data (see above)
1338 del starObs
1339 del starIds
1340 del starIndices
1341 del flagId
1342 del flagFlag
1343 del flaggedStars
1344 del refStars
1345 del refId
1346 del refMag
1347 del refMagErr
1349 # and set the bits in the cycle object
1350 fgcmFitCycle.setLUT(fgcmLut)
1351 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1352 fgcmFitCycle.setPars(fgcmPars)
1354 # finish the setup
1355 fgcmFitCycle.finishSetup()
1357 # and run
1358 fgcmFitCycle.run()
1360 ##################
1361 # Persistance
1362 ##################
1364 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1366 # Output the config for the next cycle
1367 # We need to make a copy since the input one has been frozen
1369 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1370 i, b in enumerate(_config.bands)}
1371 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1372 i, band in enumerate(_config.bands)}
1374 outConfig = copy.copy(_config)
1375 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1376 precomputeSuperStarInitialCycle=False,
1377 freezeStdAtmosphere=False,
1378 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1379 expGrayHighCutDict=updatedHighCutDict)
1381 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1382 cycleNumber=str(_config.cycleNumber + 1))
1384 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1385 outConfig.cycleNumber)
1386 outConfig.save(configFileName)
1388 if _config.isFinalCycle == 1:
1389 # We are done, ready to output products
1390 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1391 else:
1392 self.log.info("Saved config for next cycle to %s" % (configFileName))
1393 self.log.info("Be sure to look at:")
1394 self.log.info(" config.expGrayPhotometricCut")
1395 self.log.info(" config.expGrayHighCut")
1396 self.log.info("If you are satisfied with the fit, please set:")
1397 self.log.info(" config.isFinalCycle = True")
1399 return fgcmDatasetDict, outConfig
1401 def _checkDatasetsExist(self, butler):
1402 """
1403 Check if necessary datasets exist to run fgcmFitCycle
1405 Parameters
1406 ----------
1407 butler: `lsst.daf.persistence.Butler`
1409 Raises
1410 ------
1411 RuntimeError
1412 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1413 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1414 If cycleNumber > 0, then also checks for fgcmFitParameters,
1415 fgcmFlaggedStars.
1416 """
1418 if not butler.datasetExists('fgcmVisitCatalog'):
1419 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1420 if not butler.datasetExists('fgcmStarObservations'):
1421 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1422 if not butler.datasetExists('fgcmStarIds'):
1423 raise RuntimeError("Could not find fgcmStarIds in repo!")
1424 if not butler.datasetExists('fgcmStarIndices'):
1425 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1426 if not butler.datasetExists('fgcmLookUpTable'):
1427 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1429 # Need additional datasets if we are not the initial cycle
1430 if (self.config.cycleNumber > 0):
1431 if not butler.datasetExists('fgcmFitParameters',
1432 fgcmcycle=self.config.cycleNumber-1):
1433 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1434 (self.config.cycleNumber-1))
1435 if not butler.datasetExists('fgcmFlaggedStars',
1436 fgcmcycle=self.config.cycleNumber-1):
1437 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1438 (self.config.cycleNumber-1))
1440 # And additional dataset if we want reference calibration
1441 if self.config.doReferenceCalibration:
1442 if not butler.datasetExists('fgcmReferenceStars'):
1443 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1444 "doReferenceCalibration is True.")
1446 def _loadParameters(self, parCat):
1447 """
1448 Load FGCM parameters from a previous fit cycle
1450 Parameters
1451 ----------
1452 parCat : `lsst.afw.table.BaseCatalog`
1453 Parameter catalog in afw table form.
1455 Returns
1456 -------
1457 inParInfo: `numpy.ndarray`
1458 Numpy array parameter information formatted for input to fgcm
1459 inParameters: `numpy.ndarray`
1460 Numpy array parameter values formatted for input to fgcm
1461 inSuperStar: `numpy.array`
1462 Superstar flat formatted for input to fgcm
1463 """
1464 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1465 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1467 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1468 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1469 (parLutFilterNames.size, )),
1470 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1471 ('LNTAUUNIT', 'f8'),
1472 ('LNTAUSLOPEUNIT', 'f8'),
1473 ('ALPHAUNIT', 'f8'),
1474 ('LNPWVUNIT', 'f8'),
1475 ('LNPWVSLOPEUNIT', 'f8'),
1476 ('LNPWVQUADRATICUNIT', 'f8'),
1477 ('LNPWVGLOBALUNIT', 'f8'),
1478 ('O3UNIT', 'f8'),
1479 ('QESYSUNIT', 'f8'),
1480 ('FILTEROFFSETUNIT', 'f8'),
1481 ('HASEXTERNALPWV', 'i2'),
1482 ('HASEXTERNALTAU', 'i2')])
1483 inParInfo['NCCD'] = parCat['nCcd']
1484 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1485 inParInfo['FITBANDS'][:] = parFitBands
1486 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1487 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1489 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1490 ('PARO3', 'f8', (parCat['parO3'].size, )),
1491 ('PARLNTAUINTERCEPT', 'f8',
1492 (parCat['parLnTauIntercept'].size, )),
1493 ('PARLNTAUSLOPE', 'f8',
1494 (parCat['parLnTauSlope'].size, )),
1495 ('PARLNPWVINTERCEPT', 'f8',
1496 (parCat['parLnPwvIntercept'].size, )),
1497 ('PARLNPWVSLOPE', 'f8',
1498 (parCat['parLnPwvSlope'].size, )),
1499 ('PARLNPWVQUADRATIC', 'f8',
1500 (parCat['parLnPwvQuadratic'].size, )),
1501 ('PARQESYSINTERCEPT', 'f8',
1502 (parCat['parQeSysIntercept'].size, )),
1503 ('COMPQESYSSLOPE', 'f8',
1504 (parCat['compQeSysSlope'].size, )),
1505 ('PARFILTEROFFSET', 'f8',
1506 (parCat['parFilterOffset'].size, )),
1507 ('PARFILTEROFFSETFITFLAG', 'i2',
1508 (parCat['parFilterOffsetFitFlag'].size, )),
1509 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1510 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1511 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1512 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1513 ('COMPABSTHROUGHPUT', 'f8',
1514 (parCat['compAbsThroughput'].size, )),
1515 ('COMPREFOFFSET', 'f8',
1516 (parCat['compRefOffset'].size, )),
1517 ('COMPREFSIGMA', 'f8',
1518 (parCat['compRefSigma'].size, )),
1519 ('COMPMIRRORCHROMATICITY', 'f8',
1520 (parCat['compMirrorChromaticity'].size, )),
1521 ('MIRRORCHROMATICITYPIVOT', 'f8',
1522 (parCat['mirrorChromaticityPivot'].size, )),
1523 ('COMPMEDIANSEDSLOPE', 'f8',
1524 (parCat['compMedianSedSlope'].size, )),
1525 ('COMPAPERCORRPIVOT', 'f8',
1526 (parCat['compAperCorrPivot'].size, )),
1527 ('COMPAPERCORRSLOPE', 'f8',
1528 (parCat['compAperCorrSlope'].size, )),
1529 ('COMPAPERCORRSLOPEERR', 'f8',
1530 (parCat['compAperCorrSlopeErr'].size, )),
1531 ('COMPAPERCORRRANGE', 'f8',
1532 (parCat['compAperCorrRange'].size, )),
1533 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1534 (parCat['compModelErrExptimePivot'].size, )),
1535 ('COMPMODELERRFWHMPIVOT', 'f8',
1536 (parCat['compModelErrFwhmPivot'].size, )),
1537 ('COMPMODELERRSKYPIVOT', 'f8',
1538 (parCat['compModelErrSkyPivot'].size, )),
1539 ('COMPMODELERRPARS', 'f8',
1540 (parCat['compModelErrPars'].size, )),
1541 ('COMPEXPGRAY', 'f8',
1542 (parCat['compExpGray'].size, )),
1543 ('COMPVARGRAY', 'f8',
1544 (parCat['compVarGray'].size, )),
1545 ('COMPEXPDELTAMAGBKG', 'f8',
1546 (parCat['compExpDeltaMagBkg'].size, )),
1547 ('COMPNGOODSTARPEREXP', 'i4',
1548 (parCat['compNGoodStarPerExp'].size, )),
1549 ('COMPSIGFGCM', 'f8',
1550 (parCat['compSigFgcm'].size, )),
1551 ('COMPSIGMACAL', 'f8',
1552 (parCat['compSigmaCal'].size, )),
1553 ('COMPRETRIEVEDLNPWV', 'f8',
1554 (parCat['compRetrievedLnPwv'].size, )),
1555 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1556 (parCat['compRetrievedLnPwvRaw'].size, )),
1557 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1558 (parCat['compRetrievedLnPwvFlag'].size, )),
1559 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1560 (parCat['compRetrievedTauNight'].size, ))])
1562 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1563 inParams['PARO3'][:] = parCat['parO3'][0, :]
1564 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1565 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1566 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1567 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1568 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1569 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1570 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1571 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1572 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1573 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1574 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1575 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1576 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1577 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1578 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1579 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1580 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1581 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1582 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1583 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1584 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1585 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1586 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1587 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1588 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1589 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1590 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1591 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1592 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1593 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1594 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1595 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1596 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1597 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1598 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1599 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1601 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1602 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1604 return (inParInfo, inParams, inSuperStar)
1606 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1607 """
1608 Persist FGCM datasets through the butler.
1610 Parameters
1611 ----------
1612 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1613 Fgcm Fit cycle object
1614 """
1615 fgcmDatasetDict = {}
1617 # Save the parameters
1618 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1620 parSchema = afwTable.Schema()
1622 comma = ','
1623 lutFilterNameString = comma.join([n.decode('utf-8')
1624 for n in parInfo['LUTFILTERNAMES'][0]])
1625 fitBandString = comma.join([n.decode('utf-8')
1626 for n in parInfo['FITBANDS'][0]])
1628 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1629 lutFilterNameString, fitBandString)
1630 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1631 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1632 lutFilterNameString, fitBandString)
1634 fgcmDatasetDict['fgcmFitParameters'] = parCat
1636 # Save the indices of the flagged stars
1637 # (stars that have been (a) reserved from the fit for testing and
1638 # (b) bad stars that have failed quality checks.)
1639 flagStarSchema = self._makeFlagStarSchema()
1640 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1641 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1643 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1645 # Save the zeropoint information and atmospheres only if desired
1646 if self.outputZeropoints:
1647 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1648 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1650 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1651 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1653 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1655 # Save atmosphere values
1656 # These are generated by the same code that generates zeropoints
1657 atmSchema = makeAtmSchema()
1658 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1660 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1662 # Save the standard stars (if configured)
1663 if self.outputStandards:
1664 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1665 stdSchema = makeStdSchema(len(goodBands))
1666 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1668 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1670 return fgcmDatasetDict
1672 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1673 lutFilterNameString, fitBandString):
1674 """
1675 Make the parameter persistence schema
1677 Parameters
1678 ----------
1679 parInfo: `numpy.ndarray`
1680 Parameter information returned by fgcm
1681 pars: `numpy.ndarray`
1682 Parameter values returned by fgcm
1683 parSuperStarFlat: `numpy.array`
1684 Superstar flat values returned by fgcm
1685 lutFilterNameString: `str`
1686 Combined string of all the lutFilterNames
1687 fitBandString: `str`
1688 Combined string of all the fitBands
1690 Returns
1691 -------
1692 parSchema: `afwTable.schema`
1693 """
1695 parSchema = afwTable.Schema()
1697 # parameter info section
1698 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1699 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1700 size=len(lutFilterNameString))
1701 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1702 size=len(fitBandString))
1703 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1704 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1705 doc='Step units for ln(AOD) slope')
1706 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1707 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1708 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1709 doc='Step units for ln(pwv) slope')
1710 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1711 doc='Step units for ln(pwv) quadratic term')
1712 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1713 doc='Step units for global ln(pwv) parameters')
1714 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1715 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1716 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1717 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1718 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1720 # parameter section
1721 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1722 size=pars['PARALPHA'].size)
1723 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1724 size=pars['PARO3'].size)
1725 parSchema.addField('parLnTauIntercept', type='ArrayD',
1726 doc='ln(Tau) intercept parameter vector',
1727 size=pars['PARLNTAUINTERCEPT'].size)
1728 parSchema.addField('parLnTauSlope', type='ArrayD',
1729 doc='ln(Tau) slope parameter vector',
1730 size=pars['PARLNTAUSLOPE'].size)
1731 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1732 size=pars['PARLNPWVINTERCEPT'].size)
1733 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1734 size=pars['PARLNPWVSLOPE'].size)
1735 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1736 size=pars['PARLNPWVQUADRATIC'].size)
1737 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1738 size=pars['PARQESYSINTERCEPT'].size)
1739 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1740 size=pars[0]['COMPQESYSSLOPE'].size)
1741 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1742 size=pars['PARFILTEROFFSET'].size)
1743 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1744 size=pars['PARFILTEROFFSETFITFLAG'].size)
1745 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1746 doc='Global scale for retrieved ln(pwv)')
1747 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1748 doc='Global offset for retrieved ln(pwv)')
1749 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1750 doc='Nightly offset for retrieved ln(pwv)',
1751 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1752 parSchema.addField('compAbsThroughput', type='ArrayD',
1753 doc='Absolute throughput (relative to transmission curves)',
1754 size=pars['COMPABSTHROUGHPUT'].size)
1755 parSchema.addField('compRefOffset', type='ArrayD',
1756 doc='Offset between reference stars and calibrated stars',
1757 size=pars['COMPREFOFFSET'].size)
1758 parSchema.addField('compRefSigma', type='ArrayD',
1759 doc='Width of reference star/calibrated star distribution',
1760 size=pars['COMPREFSIGMA'].size)
1761 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1762 doc='Computed mirror chromaticity terms',
1763 size=pars['COMPMIRRORCHROMATICITY'].size)
1764 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1765 doc='Mirror chromaticity pivot mjd',
1766 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1767 parSchema.addField('compMedianSedSlope', type='ArrayD',
1768 doc='Computed median SED slope (per band)',
1769 size=pars['COMPMEDIANSEDSLOPE'].size)
1770 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1771 size=pars['COMPAPERCORRPIVOT'].size)
1772 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1773 size=pars['COMPAPERCORRSLOPE'].size)
1774 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1775 size=pars['COMPAPERCORRSLOPEERR'].size)
1776 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1777 size=pars['COMPAPERCORRRANGE'].size)
1778 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1779 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1780 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1781 size=pars['COMPMODELERRFWHMPIVOT'].size)
1782 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1783 size=pars['COMPMODELERRSKYPIVOT'].size)
1784 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1785 size=pars['COMPMODELERRPARS'].size)
1786 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1787 size=pars['COMPEXPGRAY'].size)
1788 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1789 size=pars['COMPVARGRAY'].size)
1790 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1791 doc='Computed exposure offset due to background',
1792 size=pars['COMPEXPDELTAMAGBKG'].size)
1793 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1794 doc='Computed number of good stars per exposure',
1795 size=pars['COMPNGOODSTARPEREXP'].size)
1796 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1797 size=pars['COMPSIGFGCM'].size)
1798 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1799 size=pars['COMPSIGMACAL'].size)
1800 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1801 size=pars['COMPRETRIEVEDLNPWV'].size)
1802 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1803 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1804 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1805 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1806 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1807 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1808 # superstarflat section
1809 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1810 size=4)
1811 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1812 size=parSuperStarFlat.size)
1814 return parSchema
1816 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1817 lutFilterNameString, fitBandString):
1818 """
1819 Make the FGCM parameter catalog for persistence
1821 Parameters
1822 ----------
1823 parSchema: `lsst.afw.table.Schema`
1824 Parameter catalog schema
1825 pars: `numpy.ndarray`
1826 FGCM parameters to put into parCat
1827 parSuperStarFlat: `numpy.array`
1828 FGCM superstar flat array to put into parCat
1829 lutFilterNameString: `str`
1830 Combined string of all the lutFilterNames
1831 fitBandString: `str`
1832 Combined string of all the fitBands
1834 Returns
1835 -------
1836 parCat: `afwTable.BasicCatalog`
1837 Atmosphere and instrumental model parameter catalog for persistence
1838 """
1840 parCat = afwTable.BaseCatalog(parSchema)
1841 parCat.reserve(1)
1843 # The parameter catalog just has one row, with many columns for all the
1844 # atmosphere and instrument fit parameters
1845 rec = parCat.addNew()
1847 # info section
1848 rec['nCcd'] = parInfo['NCCD']
1849 rec['lutFilterNames'] = lutFilterNameString
1850 rec['fitBands'] = fitBandString
1851 # note these are not currently supported here.
1852 rec['hasExternalPwv'] = 0
1853 rec['hasExternalTau'] = 0
1855 # parameter section
1857 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1859 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1860 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1861 'parQeSysIntercept', 'compQeSysSlope',
1862 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1863 'parFilterOffset', 'parFilterOffsetFitFlag',
1864 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1865 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1866 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1867 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1868 'compModelErrSkyPivot', 'compModelErrPars',
1869 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1870 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1871 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1872 'compRetrievedTauNight']
1874 for scalarName in scalarNames:
1875 rec[scalarName] = pars[scalarName.upper()]
1877 for arrName in arrNames:
1878 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1880 # superstar section
1881 rec['superstarSize'][:] = parSuperStarFlat.shape
1882 rec['superstar'][:] = parSuperStarFlat.flatten()
1884 return parCat
1886 def _makeFlagStarSchema(self):
1887 """
1888 Make the flagged-stars schema
1890 Returns
1891 -------
1892 flagStarSchema: `lsst.afw.table.Schema`
1893 """
1895 flagStarSchema = afwTable.Schema()
1897 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1898 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1900 return flagStarSchema
1902 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1903 """
1904 Make the flagged star catalog for persistence
1906 Parameters
1907 ----------
1908 flagStarSchema: `lsst.afw.table.Schema`
1909 Flagged star schema
1910 flagStarStruct: `numpy.ndarray`
1911 Flagged star structure from fgcm
1913 Returns
1914 -------
1915 flagStarCat: `lsst.afw.table.BaseCatalog`
1916 Flagged star catalog for persistence
1917 """
1919 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1920 flagStarCat.resize(flagStarStruct.size)
1922 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1923 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1925 return flagStarCat