lsst.fgcmcal g9c3108eb99+601a40e866
fgcmFitCycle.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
24
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
29
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
35
36import sys
37import traceback
38import copy
39
40import numpy as np
41
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45import lsst.afw.table as afwTable
46
47from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
48from .utilities import extractReferenceMags
49from .utilities import computeCcdOffsets, makeZptSchema, makeZptCat
50from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
51from .sedterms import SedboundarytermDict, SedtermDict
52from .utilities import lookupStaticCalibrations
53
54import fgcm
55
56__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
57
58MULTIPLE_CYCLES_MAX = 10
59
60
61class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
62 dimensions=("instrument",),
63 defaultTemplates={"previousCycleNumber": "-1",
64 "cycleNumber": "0"}):
65 camera = connectionTypes.PrerequisiteInput(
66 doc="Camera instrument",
67 name="camera",
68 storageClass="Camera",
69 dimensions=("instrument",),
70 lookupFunction=lookupStaticCalibrations,
71 isCalibration=True,
72 )
73
74 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
75 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
76 "chromatic corrections."),
77 name="fgcmLookUpTable",
78 storageClass="Catalog",
79 dimensions=("instrument",),
80 deferLoad=True,
81 )
82
83 fgcmVisitCatalog = connectionTypes.Input(
84 doc="Catalog of visit information for fgcm",
85 name="fgcmVisitCatalog",
86 storageClass="Catalog",
87 dimensions=("instrument",),
88 deferLoad=True,
89 )
90
91 fgcmStarObservations = connectionTypes.Input(
92 doc="Catalog of star observations for fgcm",
93 name="fgcmStarObservations",
94 storageClass="Catalog",
95 dimensions=("instrument",),
96 deferLoad=True,
97 )
98
99 fgcmStarIds = connectionTypes.Input(
100 doc="Catalog of fgcm calibration star IDs",
101 name="fgcmStarIds",
102 storageClass="Catalog",
103 dimensions=("instrument",),
104 deferLoad=True,
105 )
106
107 fgcmStarIndices = connectionTypes.Input(
108 doc="Catalog of fgcm calibration star indices",
109 name="fgcmStarIndices",
110 storageClass="Catalog",
111 dimensions=("instrument",),
112 deferLoad=True,
113 )
114
115 fgcmReferenceStars = connectionTypes.Input(
116 doc="Catalog of fgcm-matched reference stars",
117 name="fgcmReferenceStars",
118 storageClass="Catalog",
119 dimensions=("instrument",),
120 deferLoad=True,
121 )
122
123 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
124 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
125 name="fgcmFlaggedStars{previousCycleNumber}",
126 storageClass="Catalog",
127 dimensions=("instrument",),
128 deferLoad=True,
129 )
130
131 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
132 doc="Catalog of fgcm fit parameters from previous fit cycle",
133 name="fgcmFitParameters{previousCycleNumber}",
134 storageClass="Catalog",
135 dimensions=("instrument",),
136 deferLoad=True,
137 )
138
139 fgcmFitParameters = connectionTypes.Output(
140 doc="Catalog of fgcm fit parameters from current fit cycle",
141 name="fgcmFitParameters{cycleNumber}",
142 storageClass="Catalog",
143 dimensions=("instrument",),
144 )
145
146 fgcmFlaggedStars = connectionTypes.Output(
147 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
148 name="fgcmFlaggedStars{cycleNumber}",
149 storageClass="Catalog",
150 dimensions=("instrument",),
151 )
152
153 fgcmZeropoints = connectionTypes.Output(
154 doc="Catalog of fgcm zeropoint data from current fit cycle",
155 name="fgcmZeropoints{cycleNumber}",
156 storageClass="Catalog",
157 dimensions=("instrument",),
158 )
159
160 fgcmAtmosphereParameters = connectionTypes.Output(
161 doc="Catalog of atmospheric fit parameters from current fit cycle",
162 name="fgcmAtmosphereParameters{cycleNumber}",
163 storageClass="Catalog",
164 dimensions=("instrument",),
165 )
166
167 fgcmStandardStars = connectionTypes.Output(
168 doc="Catalog of standard star magnitudes from current fit cycle",
169 name="fgcmStandardStars{cycleNumber}",
170 storageClass="SimpleCatalog",
171 dimensions=("instrument",),
172 )
173
174 # Add connections for running multiple cycles
175 # This uses vars() to alter the class __dict__ to programmatically
176 # write many similar outputs.
177 for cycle in range(MULTIPLE_CYCLES_MAX):
178 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
179 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
180 name=f"fgcmFitParameters{cycle}",
181 storageClass="Catalog",
182 dimensions=("instrument",),
183 )
184 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
185 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
186 name=f"fgcmFlaggedStars{cycle}",
187 storageClass="Catalog",
188 dimensions=("instrument",),
189 )
190 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
191 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
192 name=f"fgcmZeropoints{cycle}",
193 storageClass="Catalog",
194 dimensions=("instrument",),
195 )
196 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
197 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
198 name=f"fgcmAtmosphereParameters{cycle}",
199 storageClass="Catalog",
200 dimensions=("instrument",),
201 )
202 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
203 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
204 name=f"fgcmStandardStars{cycle}",
205 storageClass="SimpleCatalog",
206 dimensions=("instrument",),
207 )
208
209 def __init__(self, *, config=None):
210 super().__init__(config=config)
211
212 if not config.doReferenceCalibration:
213 self.inputs.remove("fgcmReferenceStars")
214
215 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
216 raise ValueError("cycleNumber must be of integer format")
217 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
218 raise ValueError("previousCycleNumber must be of integer format")
219 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
220 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
221
222 if int(config.connections.cycleNumber) == 0:
223 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
224 self.prerequisiteInputs.remove("fgcmFitParametersInput")
225
226 if not self.config.doMultipleCycles:
227 # Single-cycle run
228 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
229 self.outputs.remove("fgcmStandardStars")
230
231 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
232 self.outputs.remove("fgcmZeropoints")
233 self.outputs.remove("fgcmAtmosphereParameters")
234
235 # Remove all multiple cycle outputs
236 for cycle in range(0, MULTIPLE_CYCLES_MAX):
237 self.outputs.remove(f"fgcmFitParameters{cycle}")
238 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
239 self.outputs.remove(f"fgcmZeropoints{cycle}")
240 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
241 self.outputs.remove(f"fgcmStandardStars{cycle}")
242
243 else:
244 # Multiple-cycle run
245 # Remove single-cycle outputs
246 self.outputs.remove("fgcmFitParameters")
247 self.outputs.remove("fgcmFlaggedStars")
248 self.outputs.remove("fgcmZeropoints")
249 self.outputs.remove("fgcmAtmosphereParameters")
250 self.outputs.remove("fgcmStandardStars")
251
252 # Remove outputs from cycles that are not used
253 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
254 MULTIPLE_CYCLES_MAX):
255 self.outputs.remove(f"fgcmFitParameters{cycle}")
256 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
257 self.outputs.remove(f"fgcmZeropoints{cycle}")
258 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
259 self.outputs.remove(f"fgcmStandardStars{cycle}")
260
261 # Remove non-final-cycle outputs if necessary
262 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
263 if not self.config.outputZeropointsBeforeFinalCycle:
264 self.outputs.remove(f"fgcmZeropoints{cycle}")
265 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
266 if not self.config.outputStandardsBeforeFinalCycle:
267 self.outputs.remove(f"fgcmStandardStars{cycle}")
268
269
270class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
271 pipelineConnections=FgcmFitCycleConnections):
272 """Config for FgcmFitCycle"""
273
274 doMultipleCycles = pexConfig.Field(
275 doc="Run multiple fit cycles in one task",
276 dtype=bool,
277 default=False,
278 )
279 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
280 doc=("Final cycle number in multiple cycle mode. The initial cycle "
281 "is 0, with limited parameters fit. The next cycle is 1 with "
282 "full parameter fit. The final cycle is a clean-up with no "
283 "parameters fit. There will be a total of "
284 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
285 "cycle number cannot be less than 2."),
286 dtype=int,
287 default=5,
288 min=2,
289 max=MULTIPLE_CYCLES_MAX,
290 inclusiveMax=True,
291 )
292 bands = pexConfig.ListField(
293 doc="Bands to run calibration",
294 dtype=str,
295 default=[],
296 )
297 fitBands = pexConfig.ListField(
298 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
299 "the atmosphere constrained from the 'fitBands' on the same night. "
300 "Must be a subset of `config.bands`"),
301 dtype=str,
302 default=[],
303 )
304 requiredBands = pexConfig.ListField(
305 doc=("Bands that are required for a star to be considered a calibration star. "
306 "Must be a subset of `config.bands`"),
307 dtype=str,
308 default=[],
309 )
310 # The following config will not be necessary after Gen2 retirement.
311 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
312 # is easiest to access in the config file.
313 physicalFilterMap = pexConfig.DictField(
314 doc="Mapping from 'physicalFilter' to band.",
315 keytype=str,
316 itemtype=str,
317 default={},
318 )
319 doReferenceCalibration = pexConfig.Field(
320 doc="Use reference catalog as additional constraint on calibration",
321 dtype=bool,
322 default=True,
323 )
324 refStarSnMin = pexConfig.Field(
325 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
326 dtype=float,
327 default=50.0,
328 )
329 refStarOutlierNSig = pexConfig.Field(
330 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
331 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
332 dtype=float,
333 default=4.0,
334 )
335 applyRefStarColorCuts = pexConfig.Field(
336 doc="Apply color cuts to reference stars?",
337 dtype=bool,
338 default=True,
339 )
340 nCore = pexConfig.Field(
341 doc="Number of cores to use",
342 dtype=int,
343 default=4,
344 )
345 nStarPerRun = pexConfig.Field(
346 doc="Number of stars to run in each chunk",
347 dtype=int,
348 default=200000,
349 )
350 nExpPerRun = pexConfig.Field(
351 doc="Number of exposures to run in each chunk",
352 dtype=int,
353 default=1000,
354 )
355 reserveFraction = pexConfig.Field(
356 doc="Fraction of stars to reserve for testing",
357 dtype=float,
358 default=0.1,
359 )
360 freezeStdAtmosphere = pexConfig.Field(
361 doc="Freeze atmosphere parameters to standard (for testing)",
362 dtype=bool,
363 default=False,
364 )
365 precomputeSuperStarInitialCycle = pexConfig.Field(
366 doc="Precompute superstar flat for initial cycle",
367 dtype=bool,
368 default=False,
369 )
370 superStarSubCcdDict = pexConfig.DictField(
371 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
372 "Must have one entry per band."),
373 keytype=str,
374 itemtype=bool,
375 default={},
376 )
377 superStarSubCcdChebyshevOrder = pexConfig.Field(
378 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
379 "Global default is first-order polynomials, and should be overridden "
380 "on a camera-by-camera basis depending on the ISR."),
381 dtype=int,
382 default=1,
383 )
384 superStarSubCcdTriangular = pexConfig.Field(
385 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
386 "suppress high-order cross terms?"),
387 dtype=bool,
388 default=False,
389 )
390 superStarSigmaClip = pexConfig.Field(
391 doc="Number of sigma to clip outliers when selecting for superstar flats",
392 dtype=float,
393 default=5.0,
394 )
395 focalPlaneSigmaClip = pexConfig.Field(
396 doc="Number of sigma to clip outliers per focal-plane.",
397 dtype=float,
398 default=4.0,
399 )
400 ccdGraySubCcdDict = pexConfig.DictField(
401 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
402 "('ccd gray') on a sub-ccd scale."),
403 keytype=str,
404 itemtype=bool,
405 default={},
406 )
407 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
408 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
409 dtype=int,
410 default=1,
411 )
412 ccdGraySubCcdTriangular = pexConfig.Field(
413 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
414 "suppress high-order cross terms?"),
415 dtype=bool,
416 default=True,
417 )
418 ccdGrayFocalPlaneDict = pexConfig.DictField(
419 doc=("Per-band specification on whether to compute focal-plane residual "
420 "('ccd gray') corrections."),
421 keytype=str,
422 itemtype=bool,
423 default={},
424 )
425 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
426 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
427 "gray corrections. If there are fewer good CCDs then the gray "
428 "correction is computed per-ccd."),
429 dtype=int,
430 default=1,
431 )
432 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
433 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
434 dtype=int,
435 default=3,
436 )
437 cycleNumber = pexConfig.Field(
438 doc=("FGCM fit cycle number. This is automatically incremented after each run "
439 "and stage of outlier rejection. See cookbook for details."),
440 dtype=int,
441 default=None,
442 )
443 isFinalCycle = pexConfig.Field(
444 doc=("Is this the final cycle of the fitting? Will automatically compute final "
445 "selection of stars and photometric exposures, and will output zeropoints "
446 "and standard stars for use in fgcmOutputProducts"),
447 dtype=bool,
448 default=False,
449 )
450 maxIterBeforeFinalCycle = pexConfig.Field(
451 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
452 "will always be 0 in the final cycle for cleanup and final selection."),
453 dtype=int,
454 default=50,
455 )
456 deltaMagBkgOffsetPercentile = pexConfig.Field(
457 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
458 "offset from local background subtraction."),
459 dtype=float,
460 default=0.25,
461 )
462 deltaMagBkgPerCcd = pexConfig.Field(
463 doc=("Compute net offset from local background subtraction per-ccd? "
464 "Otherwise, use computation per visit."),
465 dtype=bool,
466 default=False,
467 )
468 utBoundary = pexConfig.Field(
469 doc="Boundary (in UTC) from day-to-day",
470 dtype=float,
471 default=None,
472 )
473 washMjds = pexConfig.ListField(
474 doc="Mirror wash MJDs",
475 dtype=float,
476 default=(0.0,),
477 )
478 epochMjds = pexConfig.ListField(
479 doc="Epoch boundaries in MJD",
480 dtype=float,
481 default=(0.0,),
482 )
483 minObsPerBand = pexConfig.Field(
484 doc="Minimum good observations per band",
485 dtype=int,
486 default=2,
487 )
488 # TODO: When DM-16511 is done, it will be possible to get the
489 # telescope latitude directly from the camera.
490 latitude = pexConfig.Field(
491 doc="Observatory latitude",
492 dtype=float,
493 default=None,
494 )
495 brightObsGrayMax = pexConfig.Field(
496 doc="Maximum gray extinction to be considered bright observation",
497 dtype=float,
498 default=0.15,
499 )
500 minStarPerCcd = pexConfig.Field(
501 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
502 "CCDs with fewer stars will have their calibration estimated from other "
503 "CCDs in the same visit, with zeropoint error increased accordingly."),
504 dtype=int,
505 default=5,
506 )
507 minCcdPerExp = pexConfig.Field(
508 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
509 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
510 dtype=int,
511 default=5,
512 )
513 maxCcdGrayErr = pexConfig.Field(
514 doc="Maximum error on CCD gray offset to be considered photometric",
515 dtype=float,
516 default=0.05,
517 )
518 minStarPerExp = pexConfig.Field(
519 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
520 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
521 dtype=int,
522 default=600,
523 )
524 minExpPerNight = pexConfig.Field(
525 doc="Minimum number of good exposures/visits to consider a partly photometric night",
526 dtype=int,
527 default=10,
528 )
529 expGrayInitialCut = pexConfig.Field(
530 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
531 "observations."),
532 dtype=float,
533 default=-0.25,
534 )
535 expGrayPhotometricCutDict = pexConfig.DictField(
536 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
537 "('gray term') for a visit to be considered photometric. Must have one "
538 "entry per band. Broad-band filters should be -0.05."),
539 keytype=str,
540 itemtype=float,
541 default={},
542 )
543 expGrayHighCutDict = pexConfig.DictField(
544 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
545 "('gray term') for a visit to be considered photometric. Must have one "
546 "entry per band. Broad-band filters should be 0.2."),
547 keytype=str,
548 itemtype=float,
549 default={},
550 )
551 expGrayRecoverCut = pexConfig.Field(
552 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
553 "Visits with more gray extinction will only get CCD zeropoints if there are "
554 "sufficient star observations (minStarPerCcd) on that CCD."),
555 dtype=float,
556 default=-1.0,
557 )
558 expVarGrayPhotometricCutDict = pexConfig.DictField(
559 doc=("Per-band specification on maximum exposure variance to be considered possibly "
560 "photometric. Must have one entry per band. Broad-band filters should be "
561 "0.0005."),
562 keytype=str,
563 itemtype=float,
564 default={},
565 )
566 expGrayErrRecoverCut = pexConfig.Field(
567 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
568 "Visits with more gray variance will only get CCD zeropoints if there are "
569 "sufficient star observations (minStarPerCcd) on that CCD."),
570 dtype=float,
571 default=0.05,
572 )
573 aperCorrFitNBins = pexConfig.Field(
574 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
575 "no fit will be performed, and the config.aperCorrInputSlopes will be "
576 "used if available."),
577 dtype=int,
578 default=10,
579 )
580 aperCorrInputSlopeDict = pexConfig.DictField(
581 doc=("Per-band specification of aperture correction input slope parameters. These "
582 "are used on the first fit iteration, and aperture correction parameters will "
583 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
584 "to set this when there is insufficient data to fit the parameters (e.g. "
585 "tract mode)."),
586 keytype=str,
587 itemtype=float,
588 default={},
589 )
590 sedboundaryterms = pexConfig.ConfigField(
591 doc="Mapping from bands to SED boundary term names used is sedterms.",
592 dtype=SedboundarytermDict,
593 )
594 sedterms = pexConfig.ConfigField(
595 doc="Mapping from terms to bands for fgcm linear SED approximations.",
596 dtype=SedtermDict,
597 )
598 sigFgcmMaxErr = pexConfig.Field(
599 doc="Maximum mag error for fitting sigma_FGCM",
600 dtype=float,
601 default=0.01,
602 )
603 sigFgcmMaxEGrayDict = pexConfig.DictField(
604 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
605 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
606 "should be 0.05."),
607 keytype=str,
608 itemtype=float,
609 default={},
610 )
611 ccdGrayMaxStarErr = pexConfig.Field(
612 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
613 "computation"),
614 dtype=float,
615 default=0.10,
616 )
617 approxThroughputDict = pexConfig.DictField(
618 doc=("Per-band specification of the approximate overall throughput at the start of "
619 "calibration observations. Must have one entry per band. Typically should "
620 "be 1.0."),
621 keytype=str,
622 itemtype=float,
623 default={},
624 )
625 sigmaCalRange = pexConfig.ListField(
626 doc="Allowed range for systematic error floor estimation",
627 dtype=float,
628 default=(0.001, 0.003),
629 )
630 sigmaCalFitPercentile = pexConfig.ListField(
631 doc="Magnitude percentile range to fit systematic error floor",
632 dtype=float,
633 default=(0.05, 0.15),
634 )
635 sigmaCalPlotPercentile = pexConfig.ListField(
636 doc="Magnitude percentile range to plot systematic error floor",
637 dtype=float,
638 default=(0.05, 0.95),
639 )
640 sigma0Phot = pexConfig.Field(
641 doc="Systematic error floor for all zeropoints",
642 dtype=float,
643 default=0.003,
644 )
645 mapLongitudeRef = pexConfig.Field(
646 doc="Reference longitude for plotting maps",
647 dtype=float,
648 default=0.0,
649 )
650 mapNSide = pexConfig.Field(
651 doc="Healpix nside for plotting maps",
652 dtype=int,
653 default=256,
654 )
655 outfileBase = pexConfig.Field(
656 doc="Filename start for plot output files",
657 dtype=str,
658 default=None,
659 )
660 starColorCuts = pexConfig.ListField(
661 doc="Encoded star-color cuts (to be cleaned up)",
662 dtype=str,
663 default=("NO_DATA",),
664 )
665 colorSplitBands = pexConfig.ListField(
666 doc="Band names to use to split stars by color. Must have 2 entries.",
667 dtype=str,
668 length=2,
669 default=('g', 'i'),
670 )
671 modelMagErrors = pexConfig.Field(
672 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
673 dtype=bool,
674 default=True,
675 )
676 useQuadraticPwv = pexConfig.Field(
677 doc="Model PWV with a quadratic term for variation through the night?",
678 dtype=bool,
679 default=False,
680 )
681 instrumentParsPerBand = pexConfig.Field(
682 doc=("Model instrumental parameters per band? "
683 "Otherwise, instrumental parameters (QE changes with time) are "
684 "shared among all bands."),
685 dtype=bool,
686 default=False,
687 )
688 instrumentSlopeMinDeltaT = pexConfig.Field(
689 doc=("Minimum time change (in days) between observations to use in constraining "
690 "instrument slope."),
691 dtype=float,
692 default=20.0,
693 )
694 fitMirrorChromaticity = pexConfig.Field(
695 doc="Fit (intraband) mirror chromatic term?",
696 dtype=bool,
697 default=False,
698 )
699 coatingMjds = pexConfig.ListField(
700 doc="Mirror coating dates in MJD",
701 dtype=float,
702 default=(0.0,),
703 )
704 outputStandardsBeforeFinalCycle = pexConfig.Field(
705 doc="Output standard stars prior to final cycle? Used in debugging.",
706 dtype=bool,
707 default=False,
708 )
709 outputZeropointsBeforeFinalCycle = pexConfig.Field(
710 doc="Output standard stars prior to final cycle? Used in debugging.",
711 dtype=bool,
712 default=False,
713 )
714 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
715 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
716 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
717 keytype=str,
718 itemtype=bool,
719 default={},
720 )
721 autoPhotometricCutNSig = pexConfig.Field(
722 doc=("Number of sigma for automatic computation of (low) photometric cut. "
723 "Cut is based on exposure gray width (per band), unless "
724 "useRepeatabilityForExpGrayCuts is set, in which case the star "
725 "repeatability is used (also per band)."),
726 dtype=float,
727 default=3.0,
728 )
729 autoHighCutNSig = pexConfig.Field(
730 doc=("Number of sigma for automatic computation of (high) outlier cut. "
731 "Cut is based on exposure gray width (per band), unless "
732 "useRepeatabilityForExpGrayCuts is set, in which case the star "
733 "repeatability is used (also per band)."),
734 dtype=float,
735 default=4.0,
736 )
737 quietMode = pexConfig.Field(
738 doc="Be less verbose with logging.",
739 dtype=bool,
740 default=False,
741 )
742 doPlots = pexConfig.Field(
743 doc="Make fgcm QA plots.",
744 dtype=bool,
745 default=True,
746 )
747 randomSeed = pexConfig.Field(
748 doc="Random seed for fgcm for consistency in tests.",
749 dtype=int,
750 default=None,
751 optional=True,
752 )
753
754 def validate(self):
755 super().validate()
756
757 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
758 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
759 raise RuntimeError(msg)
760 if self.connections.cycleNumber != str(self.cycleNumber):
761 msg = "cycleNumber in template must be equal to connections.cycleNumber"
762 raise RuntimeError(msg)
763
764 for band in self.fitBands:
765 if band not in self.bands:
766 msg = 'fitBand %s not in bands' % (band)
767 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
768 for band in self.requiredBands:
769 if band not in self.bands:
770 msg = 'requiredBand %s not in bands' % (band)
771 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
772 for band in self.colorSplitBands:
773 if band not in self.bands:
774 msg = 'colorSplitBand %s not in bands' % (band)
775 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
776 for band in self.bands:
777 if band not in self.superStarSubCcdDict:
778 msg = 'band %s not in superStarSubCcdDict' % (band)
779 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
780 self, msg)
781 if band not in self.ccdGraySubCcdDict:
782 msg = 'band %s not in ccdGraySubCcdDict' % (band)
783 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
784 self, msg)
785 if band not in self.expGrayPhotometricCutDict:
786 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
787 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
788 self, msg)
789 if band not in self.expGrayHighCutDict:
790 msg = 'band %s not in expGrayHighCutDict' % (band)
791 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
792 self, msg)
793 if band not in self.expVarGrayPhotometricCutDict:
794 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
795 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
796 self, msg)
797 if band not in self.sigFgcmMaxEGrayDict:
798 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
799 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
800 self, msg)
801 if band not in self.approxThroughputDict:
802 msg = 'band %s not in approxThroughputDict' % (band)
803 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
804 self, msg)
805 if band not in self.useRepeatabilityForExpGrayCutsDict:
806 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
807 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
808 self, msg)
809
810
811class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
812 """Subclass of TaskRunner for fgcmFitCycleTask
813
814 fgcmFitCycleTask.run() takes one argument, the butler, and uses
815 stars and visits previously extracted from dataRefs by
816 fgcmBuildStars.
817 This Runner does not perform any dataRef parallelization, but the FGCM
818 code called by the Task uses python multiprocessing (see the "ncores"
819 config option).
820 """
821
822 @staticmethod
823 def getTargetList(parsedCmd):
824 """
825 Return a list with one element, the butler.
826 """
827 return [parsedCmd.butler]
828
829 def __call__(self, butler):
830 """
831 Parameters
832 ----------
833 butler: `lsst.daf.persistence.Butler`
834
835 Returns
836 -------
837 exitStatus: `list` with `pipeBase.Struct`
838 exitStatus (0: success; 1: failure)
839 """
840
841 task = self.TaskClass(config=self.config, log=self.log)
842
843 exitStatus = 0
844 if self.doRaise:
845 task.runDataRef(butler)
846 else:
847 try:
848 task.runDataRef(butler)
849 except Exception as e:
850 exitStatus = 1
851 task.log.fatal("Failed: %s" % e)
852 if not isinstance(e, pipeBase.TaskError):
853 traceback.print_exc(file=sys.stderr)
854
855 task.writeMetadata(butler)
856
857 # The task does not return any results:
858 return [pipeBase.Struct(exitStatus=exitStatus)]
859
860 def run(self, parsedCmd):
861 """
862 Run the task, with no multiprocessing
863
864 Parameters
865 ----------
866 parsedCmd: ArgumentParser parsed command line
867 """
868
869 resultList = []
870
871 if self.precall(parsedCmd):
872 targetList = self.getTargetList(parsedCmd)
873 # make sure that we only get 1
874 resultList = self(targetList[0])
875
876 return resultList
877
878
879class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
880 """
881 Run Single fit cycle for FGCM global calibration
882 """
883
884 ConfigClass = FgcmFitCycleConfig
885 RunnerClass = FgcmFitCycleRunner
886 _DefaultName = "fgcmFitCycle"
887
888 def __init__(self, butler=None, initInputs=None, **kwargs):
889 super().__init__(**kwargs)
890
891 # no saving of metadata for now
892 def _getMetadataName(self):
893 return None
894
895 def runQuantum(self, butlerQC, inputRefs, outputRefs):
896 camera = butlerQC.get(inputRefs.camera)
897
898 dataRefDict = {}
899
900 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
901 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
902 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
903 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
904 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
905 if self.config.doReferenceCalibration:
906 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
907 if self.config.cycleNumber > 0:
908 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
909 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
910
911 fgcmDatasetDict = None
912 if self.config.doMultipleCycles:
913 # Run multiple cycles at once.
914 config = copy.copy(self.config)
915 config.update(cycleNumber=0)
916 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
917 if cycle == self.config.multipleCyclesFinalCycleNumber:
918 config.update(isFinalCycle=True)
919
920 if cycle > 0:
921 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
922 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
923
924 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
925 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
926 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
927 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
928 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
929 if self.outputZeropoints:
930 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
931 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
932 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
933 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
934 if self.outputStandards:
935 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
936 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
937 else:
938 # Run a single cycle
939 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
940
941 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
942 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
943 if self.outputZeropoints:
944 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
945 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
946 if self.outputStandards:
947 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
948
949 @pipeBase.timeMethod
950 def runDataRef(self, butler):
951 """
952 Run a single fit cycle for FGCM
953
954 Parameters
955 ----------
956 butler: `lsst.daf.persistence.Butler`
957 """
958 self._checkDatasetsExist(butler)
959
960 dataRefDict = {}
961 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
962 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
963 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
964 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
965 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
966 if self.config.doReferenceCalibration:
967 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
968 if self.config.cycleNumber > 0:
969 lastCycle = self.config.cycleNumber - 1
970 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
971 fgcmcycle=lastCycle)
972 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
973 fgcmcycle=lastCycle)
974
975 camera = butler.get('camera')
976 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
977
978 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
979 fgcmcycle=self.config.cycleNumber)
980 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
981 fgcmcycle=self.config.cycleNumber)
982 if self.outputZeropoints:
983 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
984 fgcmcycle=self.config.cycleNumber)
985 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
986 fgcmcycle=self.config.cycleNumber)
987 if self.outputStandards:
988 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
989 fgcmcycle=self.config.cycleNumber)
990
991 def writeConfig(self, butler, clobber=False, doBackup=True):
992 """Write the configuration used for processing the data, or check that an existing
993 one is equal to the new one if present. This is an override of the regular
994 version from pipe_base that knows about fgcmcycle.
995
996 Parameters
997 ----------
998 butler : `lsst.daf.persistence.Butler`
999 Data butler used to write the config. The config is written to dataset type
1000 `CmdLineTask._getConfigName`.
1001 clobber : `bool`, optional
1002 A boolean flag that controls what happens if a config already has been saved:
1003 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1004 - `False`: raise `TaskError` if this config does not match the existing config.
1005 doBackup : `bool`, optional
1006 Set to `True` to backup the config files if clobbering.
1007 """
1008 configName = self._getConfigName()
1009 if configName is None:
1010 return
1011 if clobber:
1012 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1013 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1014 # this may be subject to a race condition; see #2789
1015 try:
1016 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1017 except Exception as exc:
1018 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1019 (configName, exc))
1020
1021 def logConfigMismatch(msg):
1022 self.log.fatal("Comparing configuration: %s", msg)
1023
1024 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1025 raise pipeBase.TaskError(
1026 f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1027 " must be consistent within the same output repo (override with --clobber-config)")
1028 else:
1029 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1030
1031 def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1032 """
1033 Run the fit cycle
1034
1035 Parameters
1036 ----------
1037 camera : `lsst.afw.cameraGeom.Camera`
1038 dataRefDict : `dict`
1039 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1040 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1041 dataRef dictionary with keys:
1042
1043 ``"fgcmLookUpTable"``
1044 dataRef for the FGCM look-up table.
1045 ``"fgcmVisitCatalog"``
1046 dataRef for visit summary catalog.
1047 ``"fgcmStarObservations"``
1048 dataRef for star observation catalog.
1049 ``"fgcmStarIds"``
1050 dataRef for star id catalog.
1051 ``"fgcmStarIndices"``
1052 dataRef for star index catalog.
1053 ``"fgcmReferenceStars"``
1054 dataRef for matched reference star catalog.
1055 ``"fgcmFlaggedStars"``
1056 dataRef for flagged star catalog.
1057 ``"fgcmFitParameters"``
1058 dataRef for fit parameter catalog.
1059 config : `lsst.pex.config.Config`, optional
1060 Configuration to use to override self.config.
1061
1062 Returns
1063 -------
1064 fgcmDatasetDict : `dict`
1065 Dictionary of datasets to persist.
1066 """
1067 if config is not None:
1068 _config = config
1069 else:
1070 _config = self.config
1071
1072 # Set defaults on whether to output standards and zeropoints
1073 self.maxIter = _config.maxIterBeforeFinalCycle
1074 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1075 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1076 self.resetFitParameters = True
1077
1078 if _config.isFinalCycle:
1079 # This is the final fit cycle, so we do not want to reset fit
1080 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1081 # and we always want to output standards and zeropoints
1082 self.maxIter = 0
1083 self.outputStandards = True
1084 self.outputZeropoints = True
1085 self.resetFitParameters = False
1086
1087 lutCat = dataRefDict['fgcmLookUpTable'].get()
1088 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1089 dict(_config.physicalFilterMap))
1090 del lutCat
1091
1092 configDict = makeConfigDict(_config, self.log, camera,
1093 self.maxIter, self.resetFitParameters,
1094 self.outputZeropoints,
1095 lutIndexVals[0]['FILTERNAMES'])
1096
1097 # next we need the exposure/visit information
1098 visitCat = dataRefDict['fgcmVisitCatalog'].get()
1099 fgcmExpInfo = translateVisitCatalog(visitCat)
1100 del visitCat
1101
1102 # Use the first orientation.
1103 # TODO: DM-21215 will generalize to arbitrary camera orientations
1104 ccdOffsets = computeCcdOffsets(camera, fgcmExpInfo['TELROT'][0])
1105
1106 noFitsDict = {'lutIndex': lutIndexVals,
1107 'lutStd': lutStd,
1108 'expInfo': fgcmExpInfo,
1109 'ccdOffsets': ccdOffsets}
1110
1111 # set up the fitter object
1112 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1113 noFitsDict=noFitsDict, noOutput=True)
1114
1115 # create the parameter object
1116 if (fgcmFitCycle.initialCycle):
1117 # cycle = 0, initial cycle
1118 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1119 fgcmLut,
1120 fgcmExpInfo)
1121 else:
1122 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog):
1123 parCat = dataRefDict['fgcmFitParameters']
1124 else:
1125 parCat = dataRefDict['fgcmFitParameters'].get()
1126 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1127 del parCat
1128 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1129 fgcmExpInfo,
1130 inParInfo,
1131 inParams,
1132 inSuperStar)
1133
1134 # set up the stars...
1135 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1136
1137 starObs = dataRefDict['fgcmStarObservations'].get()
1138 starIds = dataRefDict['fgcmStarIds'].get()
1139 starIndices = dataRefDict['fgcmStarIndices'].get()
1140
1141 # grab the flagged stars if available
1142 if 'fgcmFlaggedStars' in dataRefDict:
1143 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1144 flaggedStars = dataRefDict['fgcmFlaggedStars']
1145 else:
1146 flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1147 flagId = flaggedStars['objId'][:]
1148 flagFlag = flaggedStars['objFlag'][:]
1149 else:
1150 flaggedStars = None
1151 flagId = None
1152 flagFlag = None
1153
1154 if _config.doReferenceCalibration:
1155 refStars = dataRefDict['fgcmReferenceStars'].get()
1156
1157 refMag, refMagErr = extractReferenceMags(refStars,
1158 _config.bands,
1159 _config.physicalFilterMap)
1160 refId = refStars['fgcm_id'][:]
1161 else:
1162 refStars = None
1163 refId = None
1164 refMag = None
1165 refMagErr = None
1166
1167 # match star observations to visits
1168 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1169 # actually be transferred into fgcm using the indexing below.
1170 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1171
1172 # The fgcmStars.loadStars method will copy all the star information into
1173 # special shared memory objects that will not blow up the memory usage when
1174 # used with python multiprocessing. Once all the numbers are copied,
1175 # it is necessary to release all references to the objects that previously
1176 # stored the data to ensure that the garbage collector can clear the memory,
1177 # and ensure that this memory is not copied when multiprocessing kicks in.
1178
1179 # We determine the conversion from the native units (typically radians) to
1180 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1181 # numpy arrays rather than Angles, which would we approximately 600x slower.
1182 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1183
1184 fgcmStars.loadStars(fgcmPars,
1185 starObs['visit'][starIndices['obsIndex']],
1186 starObs['ccd'][starIndices['obsIndex']],
1187 starObs['ra'][starIndices['obsIndex']] * conv,
1188 starObs['dec'][starIndices['obsIndex']] * conv,
1189 starObs['instMag'][starIndices['obsIndex']],
1190 starObs['instMagErr'][starIndices['obsIndex']],
1191 fgcmExpInfo['FILTERNAME'][visitIndex],
1192 starIds['fgcm_id'][:],
1193 starIds['ra'][:],
1194 starIds['dec'][:],
1195 starIds['obsArrIndex'][:],
1196 starIds['nObs'][:],
1197 obsX=starObs['x'][starIndices['obsIndex']],
1198 obsY=starObs['y'][starIndices['obsIndex']],
1199 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1200 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1201 refID=refId,
1202 refMag=refMag,
1203 refMagErr=refMagErr,
1204 flagID=flagId,
1205 flagFlag=flagFlag,
1206 computeNobs=True)
1207
1208 # Release all references to temporary objects holding star data (see above)
1209 del starObs
1210 del starIds
1211 del starIndices
1212 del flagId
1213 del flagFlag
1214 del flaggedStars
1215 del refStars
1216 del refId
1217 del refMag
1218 del refMagErr
1219
1220 # and set the bits in the cycle object
1221 fgcmFitCycle.setLUT(fgcmLut)
1222 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1223 fgcmFitCycle.setPars(fgcmPars)
1224
1225 # finish the setup
1226 fgcmFitCycle.finishSetup()
1227
1228 # and run
1229 fgcmFitCycle.run()
1230
1231
1234
1235 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1236
1237 # Output the config for the next cycle
1238 # We need to make a copy since the input one has been frozen
1239
1240 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1241 i, b in enumerate(_config.bands)}
1242 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1243 i, band in enumerate(_config.bands)}
1244
1245 outConfig = copy.copy(_config)
1246 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1247 precomputeSuperStarInitialCycle=False,
1248 freezeStdAtmosphere=False,
1249 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1250 expGrayHighCutDict=updatedHighCutDict)
1251
1252 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1253 cycleNumber=str(_config.cycleNumber + 1))
1254
1255 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1256 outConfig.cycleNumber)
1257 outConfig.save(configFileName)
1258
1259 if _config.isFinalCycle == 1:
1260 # We are done, ready to output products
1261 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1262 else:
1263 self.log.info("Saved config for next cycle to %s" % (configFileName))
1264 self.log.info("Be sure to look at:")
1265 self.log.info(" config.expGrayPhotometricCut")
1266 self.log.info(" config.expGrayHighCut")
1267 self.log.info("If you are satisfied with the fit, please set:")
1268 self.log.info(" config.isFinalCycle = True")
1269
1270 fgcmFitCycle.freeSharedMemory()
1271
1272 return fgcmDatasetDict, outConfig
1273
1274 def _checkDatasetsExist(self, butler):
1275 """
1276 Check if necessary datasets exist to run fgcmFitCycle
1277
1278 Parameters
1279 ----------
1280 butler: `lsst.daf.persistence.Butler`
1281
1282 Raises
1283 ------
1284 RuntimeError
1285 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1286 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1287 If cycleNumber > 0, then also checks for fgcmFitParameters,
1288 fgcmFlaggedStars.
1289 """
1290
1291 if not butler.datasetExists('fgcmVisitCatalog'):
1292 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1293 if not butler.datasetExists('fgcmStarObservations'):
1294 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1295 if not butler.datasetExists('fgcmStarIds'):
1296 raise RuntimeError("Could not find fgcmStarIds in repo!")
1297 if not butler.datasetExists('fgcmStarIndices'):
1298 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1299 if not butler.datasetExists('fgcmLookUpTable'):
1300 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1301
1302 # Need additional datasets if we are not the initial cycle
1303 if (self.config.cycleNumber > 0):
1304 if not butler.datasetExists('fgcmFitParameters',
1305 fgcmcycle=self.config.cycleNumber-1):
1306 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1307 (self.config.cycleNumber-1))
1308 if not butler.datasetExists('fgcmFlaggedStars',
1309 fgcmcycle=self.config.cycleNumber-1):
1310 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1311 (self.config.cycleNumber-1))
1312
1313 # And additional dataset if we want reference calibration
1314 if self.config.doReferenceCalibration:
1315 if not butler.datasetExists('fgcmReferenceStars'):
1316 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1317 "doReferenceCalibration is True.")
1318
1319 def _loadParameters(self, parCat):
1320 """
1321 Load FGCM parameters from a previous fit cycle
1322
1323 Parameters
1324 ----------
1325 parCat : `lsst.afw.table.BaseCatalog`
1326 Parameter catalog in afw table form.
1327
1328 Returns
1329 -------
1330 inParInfo: `numpy.ndarray`
1331 Numpy array parameter information formatted for input to fgcm
1332 inParameters: `numpy.ndarray`
1333 Numpy array parameter values formatted for input to fgcm
1334 inSuperStar: `numpy.array`
1335 Superstar flat formatted for input to fgcm
1336 """
1337 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1338 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1339
1340 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1341 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1342 (parLutFilterNames.size, )),
1343 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1344 ('LNTAUUNIT', 'f8'),
1345 ('LNTAUSLOPEUNIT', 'f8'),
1346 ('ALPHAUNIT', 'f8'),
1347 ('LNPWVUNIT', 'f8'),
1348 ('LNPWVSLOPEUNIT', 'f8'),
1349 ('LNPWVQUADRATICUNIT', 'f8'),
1350 ('LNPWVGLOBALUNIT', 'f8'),
1351 ('O3UNIT', 'f8'),
1352 ('QESYSUNIT', 'f8'),
1353 ('FILTEROFFSETUNIT', 'f8'),
1354 ('HASEXTERNALPWV', 'i2'),
1355 ('HASEXTERNALTAU', 'i2')])
1356 inParInfo['NCCD'] = parCat['nCcd']
1357 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1358 inParInfo['FITBANDS'][:] = parFitBands
1359 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1360 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1361
1362 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1363 ('PARO3', 'f8', (parCat['parO3'].size, )),
1364 ('PARLNTAUINTERCEPT', 'f8',
1365 (parCat['parLnTauIntercept'].size, )),
1366 ('PARLNTAUSLOPE', 'f8',
1367 (parCat['parLnTauSlope'].size, )),
1368 ('PARLNPWVINTERCEPT', 'f8',
1369 (parCat['parLnPwvIntercept'].size, )),
1370 ('PARLNPWVSLOPE', 'f8',
1371 (parCat['parLnPwvSlope'].size, )),
1372 ('PARLNPWVQUADRATIC', 'f8',
1373 (parCat['parLnPwvQuadratic'].size, )),
1374 ('PARQESYSINTERCEPT', 'f8',
1375 (parCat['parQeSysIntercept'].size, )),
1376 ('COMPQESYSSLOPE', 'f8',
1377 (parCat['compQeSysSlope'].size, )),
1378 ('PARFILTEROFFSET', 'f8',
1379 (parCat['parFilterOffset'].size, )),
1380 ('PARFILTEROFFSETFITFLAG', 'i2',
1381 (parCat['parFilterOffsetFitFlag'].size, )),
1382 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1383 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1384 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1385 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1386 ('COMPABSTHROUGHPUT', 'f8',
1387 (parCat['compAbsThroughput'].size, )),
1388 ('COMPREFOFFSET', 'f8',
1389 (parCat['compRefOffset'].size, )),
1390 ('COMPREFSIGMA', 'f8',
1391 (parCat['compRefSigma'].size, )),
1392 ('COMPMIRRORCHROMATICITY', 'f8',
1393 (parCat['compMirrorChromaticity'].size, )),
1394 ('MIRRORCHROMATICITYPIVOT', 'f8',
1395 (parCat['mirrorChromaticityPivot'].size, )),
1396 ('COMPMEDIANSEDSLOPE', 'f8',
1397 (parCat['compMedianSedSlope'].size, )),
1398 ('COMPAPERCORRPIVOT', 'f8',
1399 (parCat['compAperCorrPivot'].size, )),
1400 ('COMPAPERCORRSLOPE', 'f8',
1401 (parCat['compAperCorrSlope'].size, )),
1402 ('COMPAPERCORRSLOPEERR', 'f8',
1403 (parCat['compAperCorrSlopeErr'].size, )),
1404 ('COMPAPERCORRRANGE', 'f8',
1405 (parCat['compAperCorrRange'].size, )),
1406 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1407 (parCat['compModelErrExptimePivot'].size, )),
1408 ('COMPMODELERRFWHMPIVOT', 'f8',
1409 (parCat['compModelErrFwhmPivot'].size, )),
1410 ('COMPMODELERRSKYPIVOT', 'f8',
1411 (parCat['compModelErrSkyPivot'].size, )),
1412 ('COMPMODELERRPARS', 'f8',
1413 (parCat['compModelErrPars'].size, )),
1414 ('COMPEXPGRAY', 'f8',
1415 (parCat['compExpGray'].size, )),
1416 ('COMPVARGRAY', 'f8',
1417 (parCat['compVarGray'].size, )),
1418 ('COMPEXPDELTAMAGBKG', 'f8',
1419 (parCat['compExpDeltaMagBkg'].size, )),
1420 ('COMPNGOODSTARPEREXP', 'i4',
1421 (parCat['compNGoodStarPerExp'].size, )),
1422 ('COMPSIGFGCM', 'f8',
1423 (parCat['compSigFgcm'].size, )),
1424 ('COMPSIGMACAL', 'f8',
1425 (parCat['compSigmaCal'].size, )),
1426 ('COMPRETRIEVEDLNPWV', 'f8',
1427 (parCat['compRetrievedLnPwv'].size, )),
1428 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1429 (parCat['compRetrievedLnPwvRaw'].size, )),
1430 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1431 (parCat['compRetrievedLnPwvFlag'].size, )),
1432 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1433 (parCat['compRetrievedTauNight'].size, ))])
1434
1435 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1436 inParams['PARO3'][:] = parCat['parO3'][0, :]
1437 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1438 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1439 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1440 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1441 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1442 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1443 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1444 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1445 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1446 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1447 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1448 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1449 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1450 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1451 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1452 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1453 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1454 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1455 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1456 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1457 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1458 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1459 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1460 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1461 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1462 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1463 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1464 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1465 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1466 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1467 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1468 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1469 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1470 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1471 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1472 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1473
1474 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1475 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1476
1477 return (inParInfo, inParams, inSuperStar)
1478
1479 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1480 """
1481 Persist FGCM datasets through the butler.
1482
1483 Parameters
1484 ----------
1485 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1486 Fgcm Fit cycle object
1487 """
1488 fgcmDatasetDict = {}
1489
1490 # Save the parameters
1491 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1492
1493 parSchema = afwTable.Schema()
1494
1495 comma = ','
1496 lutFilterNameString = comma.join([n.decode('utf-8')
1497 for n in parInfo['LUTFILTERNAMES'][0]])
1498 fitBandString = comma.join([n.decode('utf-8')
1499 for n in parInfo['FITBANDS'][0]])
1500
1501 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1502 lutFilterNameString, fitBandString)
1503 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1504 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1505 lutFilterNameString, fitBandString)
1506
1507 fgcmDatasetDict['fgcmFitParameters'] = parCat
1508
1509 # Save the indices of the flagged stars
1510 # (stars that have been (a) reserved from the fit for testing and
1511 # (b) bad stars that have failed quality checks.)
1512 flagStarSchema = self._makeFlagStarSchema()
1513 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1514 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1515
1516 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1517
1518 # Save the zeropoint information and atmospheres only if desired
1519 if self.outputZeropoints:
1520 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1521 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1522
1523 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1524 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1525
1526 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1527
1528 # Save atmosphere values
1529 # These are generated by the same code that generates zeropoints
1530 atmSchema = makeAtmSchema()
1531 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1532
1533 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1534
1535 # Save the standard stars (if configured)
1536 if self.outputStandards:
1537 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1538 stdSchema = makeStdSchema(len(goodBands))
1539 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1540
1541 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1542
1543 return fgcmDatasetDict
1544
1545 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1546 lutFilterNameString, fitBandString):
1547 """
1548 Make the parameter persistence schema
1549
1550 Parameters
1551 ----------
1552 parInfo: `numpy.ndarray`
1553 Parameter information returned by fgcm
1554 pars: `numpy.ndarray`
1555 Parameter values returned by fgcm
1556 parSuperStarFlat: `numpy.array`
1557 Superstar flat values returned by fgcm
1558 lutFilterNameString: `str`
1559 Combined string of all the lutFilterNames
1560 fitBandString: `str`
1561 Combined string of all the fitBands
1562
1563 Returns
1564 -------
1565 parSchema: `afwTable.schema`
1566 """
1567
1568 parSchema = afwTable.Schema()
1569
1570 # parameter info section
1571 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1572 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1573 size=len(lutFilterNameString))
1574 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1575 size=len(fitBandString))
1576 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1577 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1578 doc='Step units for ln(AOD) slope')
1579 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1580 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1581 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1582 doc='Step units for ln(pwv) slope')
1583 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1584 doc='Step units for ln(pwv) quadratic term')
1585 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1586 doc='Step units for global ln(pwv) parameters')
1587 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1588 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1589 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1590 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1591 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1592
1593 # parameter section
1594 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1595 size=pars['PARALPHA'].size)
1596 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1597 size=pars['PARO3'].size)
1598 parSchema.addField('parLnTauIntercept', type='ArrayD',
1599 doc='ln(Tau) intercept parameter vector',
1600 size=pars['PARLNTAUINTERCEPT'].size)
1601 parSchema.addField('parLnTauSlope', type='ArrayD',
1602 doc='ln(Tau) slope parameter vector',
1603 size=pars['PARLNTAUSLOPE'].size)
1604 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1605 size=pars['PARLNPWVINTERCEPT'].size)
1606 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1607 size=pars['PARLNPWVSLOPE'].size)
1608 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1609 size=pars['PARLNPWVQUADRATIC'].size)
1610 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1611 size=pars['PARQESYSINTERCEPT'].size)
1612 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1613 size=pars[0]['COMPQESYSSLOPE'].size)
1614 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1615 size=pars['PARFILTEROFFSET'].size)
1616 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1617 size=pars['PARFILTEROFFSETFITFLAG'].size)
1618 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1619 doc='Global scale for retrieved ln(pwv)')
1620 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1621 doc='Global offset for retrieved ln(pwv)')
1622 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1623 doc='Nightly offset for retrieved ln(pwv)',
1624 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1625 parSchema.addField('compAbsThroughput', type='ArrayD',
1626 doc='Absolute throughput (relative to transmission curves)',
1627 size=pars['COMPABSTHROUGHPUT'].size)
1628 parSchema.addField('compRefOffset', type='ArrayD',
1629 doc='Offset between reference stars and calibrated stars',
1630 size=pars['COMPREFOFFSET'].size)
1631 parSchema.addField('compRefSigma', type='ArrayD',
1632 doc='Width of reference star/calibrated star distribution',
1633 size=pars['COMPREFSIGMA'].size)
1634 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1635 doc='Computed mirror chromaticity terms',
1636 size=pars['COMPMIRRORCHROMATICITY'].size)
1637 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1638 doc='Mirror chromaticity pivot mjd',
1639 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1640 parSchema.addField('compMedianSedSlope', type='ArrayD',
1641 doc='Computed median SED slope (per band)',
1642 size=pars['COMPMEDIANSEDSLOPE'].size)
1643 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1644 size=pars['COMPAPERCORRPIVOT'].size)
1645 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1646 size=pars['COMPAPERCORRSLOPE'].size)
1647 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1648 size=pars['COMPAPERCORRSLOPEERR'].size)
1649 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1650 size=pars['COMPAPERCORRRANGE'].size)
1651 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1652 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1653 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1654 size=pars['COMPMODELERRFWHMPIVOT'].size)
1655 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1656 size=pars['COMPMODELERRSKYPIVOT'].size)
1657 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1658 size=pars['COMPMODELERRPARS'].size)
1659 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1660 size=pars['COMPEXPGRAY'].size)
1661 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1662 size=pars['COMPVARGRAY'].size)
1663 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1664 doc='Computed exposure offset due to background',
1665 size=pars['COMPEXPDELTAMAGBKG'].size)
1666 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1667 doc='Computed number of good stars per exposure',
1668 size=pars['COMPNGOODSTARPEREXP'].size)
1669 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1670 size=pars['COMPSIGFGCM'].size)
1671 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1672 size=pars['COMPSIGMACAL'].size)
1673 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1674 size=pars['COMPRETRIEVEDLNPWV'].size)
1675 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1676 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1677 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1678 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1679 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1680 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1681 # superstarflat section
1682 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1683 size=4)
1684 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1685 size=parSuperStarFlat.size)
1686
1687 return parSchema
1688
1689 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1690 lutFilterNameString, fitBandString):
1691 """
1692 Make the FGCM parameter catalog for persistence
1693
1694 Parameters
1695 ----------
1696 parSchema: `lsst.afw.table.Schema`
1697 Parameter catalog schema
1698 pars: `numpy.ndarray`
1699 FGCM parameters to put into parCat
1700 parSuperStarFlat: `numpy.array`
1701 FGCM superstar flat array to put into parCat
1702 lutFilterNameString: `str`
1703 Combined string of all the lutFilterNames
1704 fitBandString: `str`
1705 Combined string of all the fitBands
1706
1707 Returns
1708 -------
1709 parCat: `afwTable.BasicCatalog`
1710 Atmosphere and instrumental model parameter catalog for persistence
1711 """
1712
1713 parCat = afwTable.BaseCatalog(parSchema)
1714 parCat.reserve(1)
1715
1716 # The parameter catalog just has one row, with many columns for all the
1717 # atmosphere and instrument fit parameters
1718 rec = parCat.addNew()
1719
1720 # info section
1721 rec['nCcd'] = parInfo['NCCD']
1722 rec['lutFilterNames'] = lutFilterNameString
1723 rec['fitBands'] = fitBandString
1724 # note these are not currently supported here.
1725 rec['hasExternalPwv'] = 0
1726 rec['hasExternalTau'] = 0
1727
1728 # parameter section
1729
1730 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1731
1732 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1733 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1734 'parQeSysIntercept', 'compQeSysSlope',
1735 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1736 'parFilterOffset', 'parFilterOffsetFitFlag',
1737 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1738 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1739 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1740 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1741 'compModelErrSkyPivot', 'compModelErrPars',
1742 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1743 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1744 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1745 'compRetrievedTauNight']
1746
1747 for scalarName in scalarNames:
1748 rec[scalarName] = pars[scalarName.upper()]
1749
1750 for arrName in arrNames:
1751 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1752
1753 # superstar section
1754 rec['superstarSize'][:] = parSuperStarFlat.shape
1755 rec['superstar'][:] = parSuperStarFlat.flatten()
1756
1757 return parCat
1758
1759 def _makeFlagStarSchema(self):
1760 """
1761 Make the flagged-stars schema
1762
1763 Returns
1764 -------
1765 flagStarSchema: `lsst.afw.table.Schema`
1766 """
1767
1768 flagStarSchema = afwTable.Schema()
1769
1770 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1771 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1772
1773 return flagStarSchema
1774
1775 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1776 """
1777 Make the flagged star catalog for persistence
1778
1779 Parameters
1780 ----------
1781 flagStarSchema: `lsst.afw.table.Schema`
1782 Flagged star schema
1783 flagStarStruct: `numpy.ndarray`
1784 Flagged star structure from fgcm
1785
1786 Returns
1787 -------
1788 flagStarCat: `lsst.afw.table.BaseCatalog`
1789 Flagged star catalog for persistence
1790 """
1791
1792 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1793 flagStarCat.resize(flagStarStruct.size)
1794
1795 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1796 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1797
1798 return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:883
def makeStdSchema(nBands)
Definition: utilities.py:738
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:705
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:51
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:218
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:626
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:770
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:540
def computeCcdOffsets(camera, defaultOrientation)
Definition: utilities.py:399
def translateVisitCatalog(visitCat)
Definition: utilities.py:348