lsst.fgcmcal g34bb90edf1+19b6fef442
fgcmFitCycle.py
Go to the documentation of this file.
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Perform a single fit cycle of FGCM.
24
25This task runs a single "fit cycle" of fgcm. Prior to running this task
26one must run both fgcmMakeLut (to construct the atmosphere and instrumental
27look-up-table) and fgcmBuildStars (to extract visits and star observations
28for the global fit).
29
30The fgcmFitCycle is meant to be run multiple times, and is tracked by the
31'cycleNumber'. After each run of the fit cycle, diagnostic plots should
32be inspected to set parameters for outlier rejection on the following
33cycle. Please see the fgcmcal Cookbook for details.
34"""
35
36import sys
37import traceback
38import copy
39
40import numpy as np
41
42import lsst.pex.config as pexConfig
43import lsst.pipe.base as pipeBase
44from lsst.pipe.base import connectionTypes
45import lsst.afw.table as afwTable
46from lsst.utils.timer import timeMethod
47
48from .utilities import makeConfigDict, translateFgcmLut, translateVisitCatalog
49from .utilities import extractReferenceMags
50from .utilities import makeZptSchema, makeZptCat
51from .utilities import makeAtmSchema, makeAtmCat, makeStdSchema, makeStdCat
52from .sedterms import SedboundarytermDict, SedtermDict
53from .utilities import lookupStaticCalibrations
54from .focalPlaneProjector import FocalPlaneProjector
55
56import fgcm
57
58__all__ = ['FgcmFitCycleConfig', 'FgcmFitCycleTask', 'FgcmFitCycleRunner']
59
60MULTIPLE_CYCLES_MAX = 10
61
62
63class FgcmFitCycleConnections(pipeBase.PipelineTaskConnections,
64 dimensions=("instrument",),
65 defaultTemplates={"previousCycleNumber": "-1",
66 "cycleNumber": "0"}):
67 camera = connectionTypes.PrerequisiteInput(
68 doc="Camera instrument",
69 name="camera",
70 storageClass="Camera",
71 dimensions=("instrument",),
72 lookupFunction=lookupStaticCalibrations,
73 isCalibration=True,
74 )
75
76 fgcmLookUpTable = connectionTypes.PrerequisiteInput(
77 doc=("Atmosphere + instrument look-up-table for FGCM throughput and "
78 "chromatic corrections."),
79 name="fgcmLookUpTable",
80 storageClass="Catalog",
81 dimensions=("instrument",),
82 deferLoad=True,
83 )
84
85 fgcmVisitCatalog = connectionTypes.Input(
86 doc="Catalog of visit information for fgcm",
87 name="fgcmVisitCatalog",
88 storageClass="Catalog",
89 dimensions=("instrument",),
90 deferLoad=True,
91 )
92
93 fgcmStarObservations = connectionTypes.Input(
94 doc="Catalog of star observations for fgcm",
95 name="fgcmStarObservations",
96 storageClass="Catalog",
97 dimensions=("instrument",),
98 deferLoad=True,
99 )
100
101 fgcmStarIds = connectionTypes.Input(
102 doc="Catalog of fgcm calibration star IDs",
103 name="fgcmStarIds",
104 storageClass="Catalog",
105 dimensions=("instrument",),
106 deferLoad=True,
107 )
108
109 fgcmStarIndices = connectionTypes.Input(
110 doc="Catalog of fgcm calibration star indices",
111 name="fgcmStarIndices",
112 storageClass="Catalog",
113 dimensions=("instrument",),
114 deferLoad=True,
115 )
116
117 fgcmReferenceStars = connectionTypes.Input(
118 doc="Catalog of fgcm-matched reference stars",
119 name="fgcmReferenceStars",
120 storageClass="Catalog",
121 dimensions=("instrument",),
122 deferLoad=True,
123 )
124
125 fgcmFlaggedStarsInput = connectionTypes.PrerequisiteInput(
126 doc="Catalog of flagged stars for fgcm calibration from previous fit cycle",
127 name="fgcmFlaggedStars{previousCycleNumber}",
128 storageClass="Catalog",
129 dimensions=("instrument",),
130 deferLoad=True,
131 )
132
133 fgcmFitParametersInput = connectionTypes.PrerequisiteInput(
134 doc="Catalog of fgcm fit parameters from previous fit cycle",
135 name="fgcmFitParameters{previousCycleNumber}",
136 storageClass="Catalog",
137 dimensions=("instrument",),
138 deferLoad=True,
139 )
140
141 fgcmFitParameters = connectionTypes.Output(
142 doc="Catalog of fgcm fit parameters from current fit cycle",
143 name="fgcmFitParameters{cycleNumber}",
144 storageClass="Catalog",
145 dimensions=("instrument",),
146 )
147
148 fgcmFlaggedStars = connectionTypes.Output(
149 doc="Catalog of flagged stars for fgcm calibration from current fit cycle",
150 name="fgcmFlaggedStars{cycleNumber}",
151 storageClass="Catalog",
152 dimensions=("instrument",),
153 )
154
155 fgcmZeropoints = connectionTypes.Output(
156 doc="Catalog of fgcm zeropoint data from current fit cycle",
157 name="fgcmZeropoints{cycleNumber}",
158 storageClass="Catalog",
159 dimensions=("instrument",),
160 )
161
162 fgcmAtmosphereParameters = connectionTypes.Output(
163 doc="Catalog of atmospheric fit parameters from current fit cycle",
164 name="fgcmAtmosphereParameters{cycleNumber}",
165 storageClass="Catalog",
166 dimensions=("instrument",),
167 )
168
169 fgcmStandardStars = connectionTypes.Output(
170 doc="Catalog of standard star magnitudes from current fit cycle",
171 name="fgcmStandardStars{cycleNumber}",
172 storageClass="SimpleCatalog",
173 dimensions=("instrument",),
174 )
175
176 # Add connections for running multiple cycles
177 # This uses vars() to alter the class __dict__ to programmatically
178 # write many similar outputs.
179 for cycle in range(MULTIPLE_CYCLES_MAX):
180 vars()[f"fgcmFitParameters{cycle}"] = connectionTypes.Output(
181 doc=f"Catalog of fgcm fit parameters from fit cycle {cycle}",
182 name=f"fgcmFitParameters{cycle}",
183 storageClass="Catalog",
184 dimensions=("instrument",),
185 )
186 vars()[f"fgcmFlaggedStars{cycle}"] = connectionTypes.Output(
187 doc=f"Catalog of flagged stars for fgcm calibration from fit cycle {cycle}",
188 name=f"fgcmFlaggedStars{cycle}",
189 storageClass="Catalog",
190 dimensions=("instrument",),
191 )
192 vars()[f"fgcmZeropoints{cycle}"] = connectionTypes.Output(
193 doc=f"Catalog of fgcm zeropoint data from fit cycle {cycle}",
194 name=f"fgcmZeropoints{cycle}",
195 storageClass="Catalog",
196 dimensions=("instrument",),
197 )
198 vars()[f"fgcmAtmosphereParameters{cycle}"] = connectionTypes.Output(
199 doc=f"Catalog of atmospheric fit parameters from fit cycle {cycle}",
200 name=f"fgcmAtmosphereParameters{cycle}",
201 storageClass="Catalog",
202 dimensions=("instrument",),
203 )
204 vars()[f"fgcmStandardStars{cycle}"] = connectionTypes.Output(
205 doc=f"Catalog of standard star magnitudes from fit cycle {cycle}",
206 name=f"fgcmStandardStars{cycle}",
207 storageClass="SimpleCatalog",
208 dimensions=("instrument",),
209 )
210
211 def __init__(self, *, config=None):
212 super().__init__(config=config)
213
214 if not config.doReferenceCalibration:
215 self.inputs.remove("fgcmReferenceStars")
216
217 if str(int(config.connections.cycleNumber)) != config.connections.cycleNumber:
218 raise ValueError("cycleNumber must be of integer format")
219 if str(int(config.connections.previousCycleNumber)) != config.connections.previousCycleNumber:
220 raise ValueError("previousCycleNumber must be of integer format")
221 if int(config.connections.previousCycleNumber) != (int(config.connections.cycleNumber) - 1):
222 raise ValueError("previousCycleNumber must be 1 less than cycleNumber")
223
224 if int(config.connections.cycleNumber) == 0:
225 self.prerequisiteInputs.remove("fgcmFlaggedStarsInput")
226 self.prerequisiteInputs.remove("fgcmFitParametersInput")
227
228 if not self.config.doMultipleCycles:
229 # Single-cycle run
230 if not self.config.isFinalCycle and not self.config.outputStandardsBeforeFinalCycle:
231 self.outputs.remove("fgcmStandardStars")
232
233 if not self.config.isFinalCycle and not self.config.outputZeropointsBeforeFinalCycle:
234 self.outputs.remove("fgcmZeropoints")
235 self.outputs.remove("fgcmAtmosphereParameters")
236
237 # Remove all multiple cycle outputs
238 for cycle in range(0, MULTIPLE_CYCLES_MAX):
239 self.outputs.remove(f"fgcmFitParameters{cycle}")
240 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
241 self.outputs.remove(f"fgcmZeropoints{cycle}")
242 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
243 self.outputs.remove(f"fgcmStandardStars{cycle}")
244
245 else:
246 # Multiple-cycle run
247 # Remove single-cycle outputs
248 self.outputs.remove("fgcmFitParameters")
249 self.outputs.remove("fgcmFlaggedStars")
250 self.outputs.remove("fgcmZeropoints")
251 self.outputs.remove("fgcmAtmosphereParameters")
252 self.outputs.remove("fgcmStandardStars")
253
254 # Remove outputs from cycles that are not used
255 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1,
256 MULTIPLE_CYCLES_MAX):
257 self.outputs.remove(f"fgcmFitParameters{cycle}")
258 self.outputs.remove(f"fgcmFlaggedStars{cycle}")
259 self.outputs.remove(f"fgcmZeropoints{cycle}")
260 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
261 self.outputs.remove(f"fgcmStandardStars{cycle}")
262
263 # Remove non-final-cycle outputs if necessary
264 for cycle in range(self.config.multipleCyclesFinalCycleNumber):
265 if not self.config.outputZeropointsBeforeFinalCycle:
266 self.outputs.remove(f"fgcmZeropoints{cycle}")
267 self.outputs.remove(f"fgcmAtmosphereParameters{cycle}")
268 if not self.config.outputStandardsBeforeFinalCycle:
269 self.outputs.remove(f"fgcmStandardStars{cycle}")
270
271
272class FgcmFitCycleConfig(pipeBase.PipelineTaskConfig,
273 pipelineConnections=FgcmFitCycleConnections):
274 """Config for FgcmFitCycle"""
275
276 doMultipleCycles = pexConfig.Field(
277 doc="Run multiple fit cycles in one task",
278 dtype=bool,
279 default=False,
280 )
281 multipleCyclesFinalCycleNumber = pexConfig.RangeField(
282 doc=("Final cycle number in multiple cycle mode. The initial cycle "
283 "is 0, with limited parameters fit. The next cycle is 1 with "
284 "full parameter fit. The final cycle is a clean-up with no "
285 "parameters fit. There will be a total of "
286 "(multipleCycleFinalCycleNumber + 1) cycles run, and the final "
287 "cycle number cannot be less than 2."),
288 dtype=int,
289 default=5,
290 min=2,
291 max=MULTIPLE_CYCLES_MAX,
292 inclusiveMax=True,
293 )
294 bands = pexConfig.ListField(
295 doc="Bands to run calibration",
296 dtype=str,
297 default=[],
298 )
299 fitBands = pexConfig.ListField(
300 doc=("Bands to use in atmospheric fit. The bands not listed here will have "
301 "the atmosphere constrained from the 'fitBands' on the same night. "
302 "Must be a subset of `config.bands`"),
303 dtype=str,
304 default=[],
305 )
306 requiredBands = pexConfig.ListField(
307 doc=("Bands that are required for a star to be considered a calibration star. "
308 "Must be a subset of `config.bands`"),
309 dtype=str,
310 default=[],
311 )
312 # The following config will not be necessary after Gen2 retirement.
313 # In the meantime, it is set to 'filterDefinitions.filter_to_band' which
314 # is easiest to access in the config file.
315 physicalFilterMap = pexConfig.DictField(
316 doc="Mapping from 'physicalFilter' to band.",
317 keytype=str,
318 itemtype=str,
319 default={},
320 )
321 doReferenceCalibration = pexConfig.Field(
322 doc="Use reference catalog as additional constraint on calibration",
323 dtype=bool,
324 default=True,
325 )
326 refStarSnMin = pexConfig.Field(
327 doc="Reference star signal-to-noise minimum to use in calibration. Set to <=0 for no cut.",
328 dtype=float,
329 default=50.0,
330 )
331 refStarOutlierNSig = pexConfig.Field(
332 doc=("Number of sigma compared to average mag for reference star to be considered an outlier. "
333 "Computed per-band, and if it is an outlier in any band it is rejected from fits."),
334 dtype=float,
335 default=4.0,
336 )
337 applyRefStarColorCuts = pexConfig.Field(
338 doc="Apply color cuts to reference stars?",
339 dtype=bool,
340 default=True,
341 )
342 nCore = pexConfig.Field(
343 doc="Number of cores to use",
344 dtype=int,
345 default=4,
346 )
347 nStarPerRun = pexConfig.Field(
348 doc="Number of stars to run in each chunk",
349 dtype=int,
350 default=200000,
351 )
352 nExpPerRun = pexConfig.Field(
353 doc="Number of exposures to run in each chunk",
354 dtype=int,
355 default=1000,
356 )
357 reserveFraction = pexConfig.Field(
358 doc="Fraction of stars to reserve for testing",
359 dtype=float,
360 default=0.1,
361 )
362 freezeStdAtmosphere = pexConfig.Field(
363 doc="Freeze atmosphere parameters to standard (for testing)",
364 dtype=bool,
365 default=False,
366 )
367 precomputeSuperStarInitialCycle = pexConfig.Field(
368 doc="Precompute superstar flat for initial cycle",
369 dtype=bool,
370 default=False,
371 )
372 superStarSubCcdDict = pexConfig.DictField(
373 doc=("Per-band specification on whether to compute superstar flat on sub-ccd scale. "
374 "Must have one entry per band."),
375 keytype=str,
376 itemtype=bool,
377 default={},
378 )
379 superStarSubCcdChebyshevOrder = pexConfig.Field(
380 doc=("Order of the 2D chebyshev polynomials for sub-ccd superstar fit. "
381 "Global default is first-order polynomials, and should be overridden "
382 "on a camera-by-camera basis depending on the ISR."),
383 dtype=int,
384 default=1,
385 )
386 superStarSubCcdTriangular = pexConfig.Field(
387 doc=("Should the sub-ccd superstar chebyshev matrix be triangular to "
388 "suppress high-order cross terms?"),
389 dtype=bool,
390 default=False,
391 )
392 superStarSigmaClip = pexConfig.Field(
393 doc="Number of sigma to clip outliers when selecting for superstar flats",
394 dtype=float,
395 default=5.0,
396 )
397 focalPlaneSigmaClip = pexConfig.Field(
398 doc="Number of sigma to clip outliers per focal-plane.",
399 dtype=float,
400 default=4.0,
401 )
402 ccdGraySubCcdDict = pexConfig.DictField(
403 doc=("Per-band specification on whether to compute achromatic per-ccd residual "
404 "('ccd gray') on a sub-ccd scale."),
405 keytype=str,
406 itemtype=bool,
407 default={},
408 )
409 ccdGraySubCcdChebyshevOrder = pexConfig.Field(
410 doc="Order of the 2D chebyshev polynomials for sub-ccd gray fit.",
411 dtype=int,
412 default=1,
413 )
414 ccdGraySubCcdTriangular = pexConfig.Field(
415 doc=("Should the sub-ccd gray chebyshev matrix be triangular to "
416 "suppress high-order cross terms?"),
417 dtype=bool,
418 default=True,
419 )
420 ccdGrayFocalPlaneDict = pexConfig.DictField(
421 doc=("Per-band specification on whether to compute focal-plane residual "
422 "('ccd gray') corrections."),
423 keytype=str,
424 itemtype=bool,
425 default={},
426 )
427 ccdGrayFocalPlaneFitMinCcd = pexConfig.Field(
428 doc=("Minimum number of 'good' CCDs required to perform focal-plane "
429 "gray corrections. If there are fewer good CCDs then the gray "
430 "correction is computed per-ccd."),
431 dtype=int,
432 default=1,
433 )
434 ccdGrayFocalPlaneChebyshevOrder = pexConfig.Field(
435 doc="Order of the 2D chebyshev polynomials for focal plane fit.",
436 dtype=int,
437 default=3,
438 )
439 cycleNumber = pexConfig.Field(
440 doc=("FGCM fit cycle number. This is automatically incremented after each run "
441 "and stage of outlier rejection. See cookbook for details."),
442 dtype=int,
443 default=None,
444 )
445 isFinalCycle = pexConfig.Field(
446 doc=("Is this the final cycle of the fitting? Will automatically compute final "
447 "selection of stars and photometric exposures, and will output zeropoints "
448 "and standard stars for use in fgcmOutputProducts"),
449 dtype=bool,
450 default=False,
451 )
452 maxIterBeforeFinalCycle = pexConfig.Field(
453 doc=("Maximum fit iterations, prior to final cycle. The number of iterations "
454 "will always be 0 in the final cycle for cleanup and final selection."),
455 dtype=int,
456 default=50,
457 )
458 deltaMagBkgOffsetPercentile = pexConfig.Field(
459 doc=("Percentile brightest stars on a visit/ccd to use to compute net "
460 "offset from local background subtraction."),
461 dtype=float,
462 default=0.25,
463 )
464 deltaMagBkgPerCcd = pexConfig.Field(
465 doc=("Compute net offset from local background subtraction per-ccd? "
466 "Otherwise, use computation per visit."),
467 dtype=bool,
468 default=False,
469 )
470 utBoundary = pexConfig.Field(
471 doc="Boundary (in UTC) from day-to-day",
472 dtype=float,
473 default=None,
474 )
475 washMjds = pexConfig.ListField(
476 doc="Mirror wash MJDs",
477 dtype=float,
478 default=(0.0,),
479 )
480 epochMjds = pexConfig.ListField(
481 doc="Epoch boundaries in MJD",
482 dtype=float,
483 default=(0.0,),
484 )
485 minObsPerBand = pexConfig.Field(
486 doc="Minimum good observations per band",
487 dtype=int,
488 default=2,
489 )
490 # TODO: When DM-16511 is done, it will be possible to get the
491 # telescope latitude directly from the camera.
492 latitude = pexConfig.Field(
493 doc="Observatory latitude",
494 dtype=float,
495 default=None,
496 )
497 defaultCameraOrientation = pexConfig.Field(
498 doc="Default camera orientation for QA plots.",
499 dtype=float,
500 default=None,
501 )
502 brightObsGrayMax = pexConfig.Field(
503 doc="Maximum gray extinction to be considered bright observation",
504 dtype=float,
505 default=0.15,
506 )
507 minStarPerCcd = pexConfig.Field(
508 doc=("Minimum number of good stars per CCD to be used in calibration fit. "
509 "CCDs with fewer stars will have their calibration estimated from other "
510 "CCDs in the same visit, with zeropoint error increased accordingly."),
511 dtype=int,
512 default=5,
513 )
514 minCcdPerExp = pexConfig.Field(
515 doc=("Minimum number of good CCDs per exposure/visit to be used in calibration fit. "
516 "Visits with fewer good CCDs will have CCD zeropoints estimated where possible."),
517 dtype=int,
518 default=5,
519 )
520 maxCcdGrayErr = pexConfig.Field(
521 doc="Maximum error on CCD gray offset to be considered photometric",
522 dtype=float,
523 default=0.05,
524 )
525 minStarPerExp = pexConfig.Field(
526 doc=("Minimum number of good stars per exposure/visit to be used in calibration fit. "
527 "Visits with fewer good stars will have CCD zeropoints estimated where possible."),
528 dtype=int,
529 default=600,
530 )
531 minExpPerNight = pexConfig.Field(
532 doc="Minimum number of good exposures/visits to consider a partly photometric night",
533 dtype=int,
534 default=10,
535 )
536 expGrayInitialCut = pexConfig.Field(
537 doc=("Maximum exposure/visit gray value for initial selection of possible photometric "
538 "observations."),
539 dtype=float,
540 default=-0.25,
541 )
542 expGrayPhotometricCutDict = pexConfig.DictField(
543 doc=("Per-band specification on maximum (negative) achromatic exposure residual "
544 "('gray term') for a visit to be considered photometric. Must have one "
545 "entry per band. Broad-band filters should be -0.05."),
546 keytype=str,
547 itemtype=float,
548 default={},
549 )
550 expGrayHighCutDict = pexConfig.DictField(
551 doc=("Per-band specification on maximum (positive) achromatic exposure residual "
552 "('gray term') for a visit to be considered photometric. Must have one "
553 "entry per band. Broad-band filters should be 0.2."),
554 keytype=str,
555 itemtype=float,
556 default={},
557 )
558 expGrayRecoverCut = pexConfig.Field(
559 doc=("Maximum (negative) exposure gray to be able to recover bad ccds via interpolation. "
560 "Visits with more gray extinction will only get CCD zeropoints if there are "
561 "sufficient star observations (minStarPerCcd) on that CCD."),
562 dtype=float,
563 default=-1.0,
564 )
565 expVarGrayPhotometricCutDict = pexConfig.DictField(
566 doc=("Per-band specification on maximum exposure variance to be considered possibly "
567 "photometric. Must have one entry per band. Broad-band filters should be "
568 "0.0005."),
569 keytype=str,
570 itemtype=float,
571 default={},
572 )
573 expGrayErrRecoverCut = pexConfig.Field(
574 doc=("Maximum exposure gray error to be able to recover bad ccds via interpolation. "
575 "Visits with more gray variance will only get CCD zeropoints if there are "
576 "sufficient star observations (minStarPerCcd) on that CCD."),
577 dtype=float,
578 default=0.05,
579 )
580 aperCorrFitNBins = pexConfig.Field(
581 doc=("Number of aperture bins used in aperture correction fit. When set to 0"
582 "no fit will be performed, and the config.aperCorrInputSlopes will be "
583 "used if available."),
584 dtype=int,
585 default=10,
586 )
587 aperCorrInputSlopeDict = pexConfig.DictField(
588 doc=("Per-band specification of aperture correction input slope parameters. These "
589 "are used on the first fit iteration, and aperture correction parameters will "
590 "be updated from the data if config.aperCorrFitNBins > 0. It is recommended "
591 "to set this when there is insufficient data to fit the parameters (e.g. "
592 "tract mode)."),
593 keytype=str,
594 itemtype=float,
595 default={},
596 )
597 sedboundaryterms = pexConfig.ConfigField(
598 doc="Mapping from bands to SED boundary term names used is sedterms.",
599 dtype=SedboundarytermDict,
600 )
601 sedterms = pexConfig.ConfigField(
602 doc="Mapping from terms to bands for fgcm linear SED approximations.",
603 dtype=SedtermDict,
604 )
605 sigFgcmMaxErr = pexConfig.Field(
606 doc="Maximum mag error for fitting sigma_FGCM",
607 dtype=float,
608 default=0.01,
609 )
610 sigFgcmMaxEGrayDict = pexConfig.DictField(
611 doc=("Per-band specification for maximum (absolute) achromatic residual (gray value) "
612 "for observations in sigma_fgcm (raw repeatability). Broad-band filters "
613 "should be 0.05."),
614 keytype=str,
615 itemtype=float,
616 default={},
617 )
618 ccdGrayMaxStarErr = pexConfig.Field(
619 doc=("Maximum error on a star observation to use in ccd gray (achromatic residual) "
620 "computation"),
621 dtype=float,
622 default=0.10,
623 )
624 approxThroughputDict = pexConfig.DictField(
625 doc=("Per-band specification of the approximate overall throughput at the start of "
626 "calibration observations. Must have one entry per band. Typically should "
627 "be 1.0."),
628 keytype=str,
629 itemtype=float,
630 default={},
631 )
632 sigmaCalRange = pexConfig.ListField(
633 doc="Allowed range for systematic error floor estimation",
634 dtype=float,
635 default=(0.001, 0.003),
636 )
637 sigmaCalFitPercentile = pexConfig.ListField(
638 doc="Magnitude percentile range to fit systematic error floor",
639 dtype=float,
640 default=(0.05, 0.15),
641 )
642 sigmaCalPlotPercentile = pexConfig.ListField(
643 doc="Magnitude percentile range to plot systematic error floor",
644 dtype=float,
645 default=(0.05, 0.95),
646 )
647 sigma0Phot = pexConfig.Field(
648 doc="Systematic error floor for all zeropoints",
649 dtype=float,
650 default=0.003,
651 )
652 mapLongitudeRef = pexConfig.Field(
653 doc="Reference longitude for plotting maps",
654 dtype=float,
655 default=0.0,
656 )
657 mapNSide = pexConfig.Field(
658 doc="Healpix nside for plotting maps",
659 dtype=int,
660 default=256,
661 )
662 outfileBase = pexConfig.Field(
663 doc="Filename start for plot output files",
664 dtype=str,
665 default=None,
666 )
667 starColorCuts = pexConfig.ListField(
668 doc="Encoded star-color cuts (to be cleaned up)",
669 dtype=str,
670 default=("NO_DATA",),
671 )
672 colorSplitBands = pexConfig.ListField(
673 doc="Band names to use to split stars by color. Must have 2 entries.",
674 dtype=str,
675 length=2,
676 default=('g', 'i'),
677 )
678 modelMagErrors = pexConfig.Field(
679 doc="Should FGCM model the magnitude errors from sky/fwhm? (False means trust inputs)",
680 dtype=bool,
681 default=True,
682 )
683 useQuadraticPwv = pexConfig.Field(
684 doc="Model PWV with a quadratic term for variation through the night?",
685 dtype=bool,
686 default=False,
687 )
688 instrumentParsPerBand = pexConfig.Field(
689 doc=("Model instrumental parameters per band? "
690 "Otherwise, instrumental parameters (QE changes with time) are "
691 "shared among all bands."),
692 dtype=bool,
693 default=False,
694 )
695 instrumentSlopeMinDeltaT = pexConfig.Field(
696 doc=("Minimum time change (in days) between observations to use in constraining "
697 "instrument slope."),
698 dtype=float,
699 default=20.0,
700 )
701 fitMirrorChromaticity = pexConfig.Field(
702 doc="Fit (intraband) mirror chromatic term?",
703 dtype=bool,
704 default=False,
705 )
706 coatingMjds = pexConfig.ListField(
707 doc="Mirror coating dates in MJD",
708 dtype=float,
709 default=(0.0,),
710 )
711 outputStandardsBeforeFinalCycle = pexConfig.Field(
712 doc="Output standard stars prior to final cycle? Used in debugging.",
713 dtype=bool,
714 default=False,
715 )
716 outputZeropointsBeforeFinalCycle = pexConfig.Field(
717 doc="Output standard stars prior to final cycle? Used in debugging.",
718 dtype=bool,
719 default=False,
720 )
721 useRepeatabilityForExpGrayCutsDict = pexConfig.DictField(
722 doc=("Per-band specification on whether to use star repeatability (instead of exposures) "
723 "for computing photometric cuts. Recommended for tract mode or bands with few visits."),
724 keytype=str,
725 itemtype=bool,
726 default={},
727 )
728 autoPhotometricCutNSig = pexConfig.Field(
729 doc=("Number of sigma for automatic computation of (low) photometric cut. "
730 "Cut is based on exposure gray width (per band), unless "
731 "useRepeatabilityForExpGrayCuts is set, in which case the star "
732 "repeatability is used (also per band)."),
733 dtype=float,
734 default=3.0,
735 )
736 autoHighCutNSig = pexConfig.Field(
737 doc=("Number of sigma for automatic computation of (high) outlier cut. "
738 "Cut is based on exposure gray width (per band), unless "
739 "useRepeatabilityForExpGrayCuts is set, in which case the star "
740 "repeatability is used (also per band)."),
741 dtype=float,
742 default=4.0,
743 )
744 quietMode = pexConfig.Field(
745 doc="Be less verbose with logging.",
746 dtype=bool,
747 default=False,
748 )
749 doPlots = pexConfig.Field(
750 doc="Make fgcm QA plots.",
751 dtype=bool,
752 default=True,
753 )
754 randomSeed = pexConfig.Field(
755 doc="Random seed for fgcm for consistency in tests.",
756 dtype=int,
757 default=None,
758 optional=True,
759 )
760 deltaAperFitMinNgoodObs = pexConfig.Field(
761 doc="Minimum number of good observations to use mean delta-aper values in fits.",
762 dtype=int,
763 default=2,
764 )
765 deltaAperFitPerCcdNx = pexConfig.Field(
766 doc=("Number of x bins per ccd when computing delta-aper background offsets. "
767 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
768 dtype=int,
769 default=10,
770 )
771 deltaAperFitPerCcdNy = pexConfig.Field(
772 doc=("Number of y bins per ccd when computing delta-aper background offsets. "
773 "Only used when ``doComputeDeltaAperPerCcd`` is True."),
774 dtype=int,
775 default=10,
776 )
777 deltaAperFitSpatialNside = pexConfig.Field(
778 doc="Healpix nside to compute spatial delta-aper background offset maps.",
779 dtype=int,
780 default=64,
781 )
782 deltaAperInnerRadiusArcsec = pexConfig.Field(
783 doc=("Inner radius used to compute deltaMagAper (arcseconds). "
784 "Must be positive and less than ``deltaAperOuterRadiusArcsec`` if "
785 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
786 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
787 dtype=float,
788 default=0.0,
789 )
790 deltaAperOuterRadiusArcsec = pexConfig.Field(
791 doc=("Outer radius used to compute deltaMagAper (arcseconds). "
792 "Must be positive and greater than ``deltaAperInnerRadiusArcsec`` if "
793 "any of ``doComputeDeltaAperPerVisit``, ``doComputeDeltaAperPerStar``, "
794 "``doComputeDeltaAperMap``, ``doComputeDeltaAperPerCcd`` are set."),
795 dtype=float,
796 default=0.0,
797 )
798 doComputeDeltaAperPerVisit = pexConfig.Field(
799 doc=("Do the computation of delta-aper background offsets per visit? "
800 "Note: this option can be very slow when there are many visits."),
801 dtype=bool,
802 default=False,
803 )
804 doComputeDeltaAperPerStar = pexConfig.Field(
805 doc="Do the computation of delta-aper mean values per star?",
806 dtype=bool,
807 default=True,
808 )
809 doComputeDeltaAperMap = pexConfig.Field(
810 doc=("Do the computation of delta-aper spatial maps? "
811 "This is only used if ``doComputeDeltaAperPerStar`` is True,"),
812 dtype=bool,
813 default=False,
814 )
815 doComputeDeltaAperPerCcd = pexConfig.Field(
816 doc="Do the computation of per-ccd delta-aper background offsets?",
817 dtype=bool,
818 default=False,
819 )
820
821 def validate(self):
822 super().validate()
823
824 if self.connections.previousCycleNumber != str(self.cycleNumber - 1):
825 msg = "cycleNumber in template must be connections.previousCycleNumber + 1"
826 raise RuntimeError(msg)
827 if self.connections.cycleNumber != str(self.cycleNumber):
828 msg = "cycleNumber in template must be equal to connections.cycleNumber"
829 raise RuntimeError(msg)
830
831 for band in self.fitBands:
832 if band not in self.bands:
833 msg = 'fitBand %s not in bands' % (band)
834 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.fitBands, self, msg)
835 for band in self.requiredBands:
836 if band not in self.bands:
837 msg = 'requiredBand %s not in bands' % (band)
838 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.requiredBands, self, msg)
839 for band in self.colorSplitBands:
840 if band not in self.bands:
841 msg = 'colorSplitBand %s not in bands' % (band)
842 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.colorSplitBands, self, msg)
843 for band in self.bands:
844 if band not in self.superStarSubCcdDict:
845 msg = 'band %s not in superStarSubCcdDict' % (band)
846 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.superStarSubCcdDict,
847 self, msg)
848 if band not in self.ccdGraySubCcdDict:
849 msg = 'band %s not in ccdGraySubCcdDict' % (band)
850 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.ccdGraySubCcdDict,
851 self, msg)
852 if band not in self.expGrayPhotometricCutDict:
853 msg = 'band %s not in expGrayPhotometricCutDict' % (band)
854 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayPhotometricCutDict,
855 self, msg)
856 if band not in self.expGrayHighCutDict:
857 msg = 'band %s not in expGrayHighCutDict' % (band)
858 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expGrayHighCutDict,
859 self, msg)
860 if band not in self.expVarGrayPhotometricCutDict:
861 msg = 'band %s not in expVarGrayPhotometricCutDict' % (band)
862 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.expVarGrayPhotometricCutDict,
863 self, msg)
864 if band not in self.sigFgcmMaxEGrayDict:
865 msg = 'band %s not in sigFgcmMaxEGrayDict' % (band)
866 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.sigFgcmMaxEGrayDict,
867 self, msg)
868 if band not in self.approxThroughputDict:
869 msg = 'band %s not in approxThroughputDict' % (band)
870 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.approxThroughputDict,
871 self, msg)
872 if band not in self.useRepeatabilityForExpGrayCutsDict:
873 msg = 'band %s not in useRepeatabilityForExpGrayCutsDict' % (band)
874 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
875 self, msg)
876
877 if self.doComputeDeltaAperPerVisit or self.doComputeDeltaAperMap \
878 or self.doComputeDeltaAperPerCcd:
879 if self.deltaAperInnerRadiusArcsec <= 0.0:
880 msg = 'deltaAperInnerRadiusArcsec must be positive if deltaAper computations are turned on.'
881 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperInnerRadiusArcsec,
882 self, msg)
883 if self.deltaAperOuterRadiusArcsec <= 0.0:
884 msg = 'deltaAperOuterRadiusArcsec must be positive if deltaAper computations are turned on.'
885 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
886 self, msg)
887 if self.deltaAperOuterRadiusArcsec <= self.deltaAperInnerRadiusArcsec:
888 msg = ('deltaAperOuterRadiusArcsec must be greater than deltaAperInnerRadiusArcsec if '
889 'deltaAper computations are turned on.')
890 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.deltaAperOuterRadiusArcsec,
891 self, msg)
892
893
894class FgcmFitCycleRunner(pipeBase.ButlerInitializedTaskRunner):
895 """Subclass of TaskRunner for fgcmFitCycleTask
896
897 fgcmFitCycleTask.run() takes one argument, the butler, and uses
898 stars and visits previously extracted from dataRefs by
899 fgcmBuildStars.
900 This Runner does not perform any dataRef parallelization, but the FGCM
901 code called by the Task uses python multiprocessing (see the "ncores"
902 config option).
903 """
904
905 @staticmethod
906 def getTargetList(parsedCmd):
907 """
908 Return a list with one element, the butler.
909 """
910 return [parsedCmd.butler]
911
912 def __call__(self, butler):
913 """
914 Parameters
915 ----------
916 butler: `lsst.daf.persistence.Butler`
917
918 Returns
919 -------
920 exitStatus: `list` with `pipeBase.Struct`
921 exitStatus (0: success; 1: failure)
922 """
923
924 task = self.TaskClass(config=self.config, log=self.log)
925
926 exitStatus = 0
927 if self.doRaise:
928 task.runDataRef(butler)
929 else:
930 try:
931 task.runDataRef(butler)
932 except Exception as e:
933 exitStatus = 1
934 task.log.fatal("Failed: %s" % e)
935 if not isinstance(e, pipeBase.TaskError):
936 traceback.print_exc(file=sys.stderr)
937
938 task.writeMetadata(butler)
939
940 # The task does not return any results:
941 return [pipeBase.Struct(exitStatus=exitStatus)]
942
943 def run(self, parsedCmd):
944 """
945 Run the task, with no multiprocessing
946
947 Parameters
948 ----------
949 parsedCmd: ArgumentParser parsed command line
950 """
951
952 resultList = []
953
954 if self.precall(parsedCmd):
955 targetList = self.getTargetList(parsedCmd)
956 # make sure that we only get 1
957 resultList = self(targetList[0])
958
959 return resultList
960
961
962class FgcmFitCycleTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
963 """
964 Run Single fit cycle for FGCM global calibration
965 """
966
967 ConfigClass = FgcmFitCycleConfig
968 RunnerClass = FgcmFitCycleRunner
969 _DefaultName = "fgcmFitCycle"
970
971 def __init__(self, butler=None, initInputs=None, **kwargs):
972 super().__init__(**kwargs)
973
974 # no saving of metadata for now
975 def _getMetadataName(self):
976 return None
977
978 def runQuantum(self, butlerQC, inputRefs, outputRefs):
979 camera = butlerQC.get(inputRefs.camera)
980
981 dataRefDict = {}
982
983 dataRefDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
984 dataRefDict['fgcmVisitCatalog'] = butlerQC.get(inputRefs.fgcmVisitCatalog)
985 dataRefDict['fgcmStarObservations'] = butlerQC.get(inputRefs.fgcmStarObservations)
986 dataRefDict['fgcmStarIds'] = butlerQC.get(inputRefs.fgcmStarIds)
987 dataRefDict['fgcmStarIndices'] = butlerQC.get(inputRefs.fgcmStarIndices)
988 if self.config.doReferenceCalibration:
989 dataRefDict['fgcmReferenceStars'] = butlerQC.get(inputRefs.fgcmReferenceStars)
990 if self.config.cycleNumber > 0:
991 dataRefDict['fgcmFlaggedStars'] = butlerQC.get(inputRefs.fgcmFlaggedStarsInput)
992 dataRefDict['fgcmFitParameters'] = butlerQC.get(inputRefs.fgcmFitParametersInput)
993
994 fgcmDatasetDict = None
995 if self.config.doMultipleCycles:
996 # Run multiple cycles at once.
997 config = copy.copy(self.config)
998 config.update(cycleNumber=0)
999 for cycle in range(self.config.multipleCyclesFinalCycleNumber + 1):
1000 if cycle == self.config.multipleCyclesFinalCycleNumber:
1001 config.update(isFinalCycle=True)
1002
1003 if cycle > 0:
1004 dataRefDict['fgcmFlaggedStars'] = fgcmDatasetDict['fgcmFlaggedStars']
1005 dataRefDict['fgcmFitParameters'] = fgcmDatasetDict['fgcmFitParameters']
1006
1007 fgcmDatasetDict, config = self._fgcmFitCycle(camera, dataRefDict, config=config)
1008 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'],
1009 getattr(outputRefs, f'fgcmFitParameters{cycle}'))
1010 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'],
1011 getattr(outputRefs, f'fgcmFlaggedStars{cycle}'))
1012 if self.outputZeropoints:
1013 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'],
1014 getattr(outputRefs, f'fgcmZeropoints{cycle}'))
1015 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'],
1016 getattr(outputRefs, f'fgcmAtmosphereParameters{cycle}'))
1017 if self.outputStandards:
1018 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'],
1019 getattr(outputRefs, f'fgcmStandardStars{cycle}'))
1020 else:
1021 # Run a single cycle
1022 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
1023
1024 butlerQC.put(fgcmDatasetDict['fgcmFitParameters'], outputRefs.fgcmFitParameters)
1025 butlerQC.put(fgcmDatasetDict['fgcmFlaggedStars'], outputRefs.fgcmFlaggedStars)
1026 if self.outputZeropoints:
1027 butlerQC.put(fgcmDatasetDict['fgcmZeropoints'], outputRefs.fgcmZeropoints)
1028 butlerQC.put(fgcmDatasetDict['fgcmAtmosphereParameters'], outputRefs.fgcmAtmosphereParameters)
1029 if self.outputStandards:
1030 butlerQC.put(fgcmDatasetDict['fgcmStandardStars'], outputRefs.fgcmStandardStars)
1031
1032 @timeMethod
1033 def runDataRef(self, butler):
1034 """
1035 Run a single fit cycle for FGCM
1036
1037 Parameters
1038 ----------
1039 butler: `lsst.daf.persistence.Butler`
1040 """
1041 self._checkDatasetsExist(butler)
1042
1043 dataRefDict = {}
1044 dataRefDict['fgcmLookUpTable'] = butler.dataRef('fgcmLookUpTable')
1045 dataRefDict['fgcmVisitCatalog'] = butler.dataRef('fgcmVisitCatalog')
1046 dataRefDict['fgcmStarObservations'] = butler.dataRef('fgcmStarObservations')
1047 dataRefDict['fgcmStarIds'] = butler.dataRef('fgcmStarIds')
1048 dataRefDict['fgcmStarIndices'] = butler.dataRef('fgcmStarIndices')
1049 if self.config.doReferenceCalibration:
1050 dataRefDict['fgcmReferenceStars'] = butler.dataRef('fgcmReferenceStars')
1051 if self.config.cycleNumber > 0:
1052 lastCycle = self.config.cycleNumber - 1
1053 dataRefDict['fgcmFlaggedStars'] = butler.dataRef('fgcmFlaggedStars',
1054 fgcmcycle=lastCycle)
1055 dataRefDict['fgcmFitParameters'] = butler.dataRef('fgcmFitParameters',
1056 fgcmcycle=lastCycle)
1057
1058 camera = butler.get('camera')
1059 fgcmDatasetDict, _ = self._fgcmFitCycle(camera, dataRefDict)
1060
1061 butler.put(fgcmDatasetDict['fgcmFitParameters'], 'fgcmFitParameters',
1062 fgcmcycle=self.config.cycleNumber)
1063 butler.put(fgcmDatasetDict['fgcmFlaggedStars'], 'fgcmFlaggedStars',
1064 fgcmcycle=self.config.cycleNumber)
1065 if self.outputZeropoints:
1066 butler.put(fgcmDatasetDict['fgcmZeropoints'], 'fgcmZeropoints',
1067 fgcmcycle=self.config.cycleNumber)
1068 butler.put(fgcmDatasetDict['fgcmAtmosphereParameters'], 'fgcmAtmosphereParameters',
1069 fgcmcycle=self.config.cycleNumber)
1070 if self.outputStandards:
1071 butler.put(fgcmDatasetDict['fgcmStandardStars'], 'fgcmStandardStars',
1072 fgcmcycle=self.config.cycleNumber)
1073
1074 def writeConfig(self, butler, clobber=False, doBackup=True):
1075 """Write the configuration used for processing the data, or check that an existing
1076 one is equal to the new one if present. This is an override of the regular
1077 version from pipe_base that knows about fgcmcycle.
1078
1079 Parameters
1080 ----------
1081 butler : `lsst.daf.persistence.Butler`
1082 Data butler used to write the config. The config is written to dataset type
1083 `CmdLineTask._getConfigName`.
1084 clobber : `bool`, optional
1085 A boolean flag that controls what happens if a config already has been saved:
1086 - `True`: overwrite or rename the existing config, depending on ``doBackup``.
1087 - `False`: raise `TaskError` if this config does not match the existing config.
1088 doBackup : `bool`, optional
1089 Set to `True` to backup the config files if clobbering.
1090 """
1091 configName = self._getConfigName()
1092 if configName is None:
1093 return
1094 if clobber:
1095 butler.put(self.config, configName, doBackup=doBackup, fgcmcycle=self.config.cycleNumber)
1096 elif butler.datasetExists(configName, write=True, fgcmcycle=self.config.cycleNumber):
1097 # this may be subject to a race condition; see #2789
1098 try:
1099 oldConfig = butler.get(configName, immediate=True, fgcmcycle=self.config.cycleNumber)
1100 except Exception as exc:
1101 raise type(exc)("Unable to read stored config file %s (%s); consider using --clobber-config" %
1102 (configName, exc))
1103
1104 def logConfigMismatch(msg):
1105 self.log.fatal("Comparing configuration: %s", msg)
1106
1107 if not self.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
1108 raise pipeBase.TaskError(
1109 f"Config does not match existing task config {configName!r} on disk; tasks configurations"
1110 " must be consistent within the same output repo (override with --clobber-config)")
1111 else:
1112 butler.put(self.config, configName, fgcmcycle=self.config.cycleNumber)
1113
1114 def _fgcmFitCycle(self, camera, dataRefDict, config=None):
1115 """
1116 Run the fit cycle
1117
1118 Parameters
1119 ----------
1120 camera : `lsst.afw.cameraGeom.Camera`
1121 dataRefDict : `dict`
1122 All dataRefs are `lsst.daf.persistence.ButlerDataRef` (gen2) or
1123 `lsst.daf.butler.DeferredDatasetHandle` (gen3)
1124 dataRef dictionary with keys:
1125
1126 ``"fgcmLookUpTable"``
1127 dataRef for the FGCM look-up table.
1128 ``"fgcmVisitCatalog"``
1129 dataRef for visit summary catalog.
1130 ``"fgcmStarObservations"``
1131 dataRef for star observation catalog.
1132 ``"fgcmStarIds"``
1133 dataRef for star id catalog.
1134 ``"fgcmStarIndices"``
1135 dataRef for star index catalog.
1136 ``"fgcmReferenceStars"``
1137 dataRef for matched reference star catalog.
1138 ``"fgcmFlaggedStars"``
1139 dataRef for flagged star catalog.
1140 ``"fgcmFitParameters"``
1141 dataRef for fit parameter catalog.
1142 config : `lsst.pex.config.Config`, optional
1143 Configuration to use to override self.config.
1144
1145 Returns
1146 -------
1147 fgcmDatasetDict : `dict`
1148 Dictionary of datasets to persist.
1149 """
1150 if config is not None:
1151 _config = config
1152 else:
1153 _config = self.config
1154
1155 # Set defaults on whether to output standards and zeropoints
1156 self.maxIter = _config.maxIterBeforeFinalCycle
1157 self.outputStandards = _config.outputStandardsBeforeFinalCycle
1158 self.outputZeropoints = _config.outputZeropointsBeforeFinalCycle
1159 self.resetFitParameters = True
1160
1161 if _config.isFinalCycle:
1162 # This is the final fit cycle, so we do not want to reset fit
1163 # parameters, we want to run a final "clean-up" with 0 fit iterations,
1164 # and we always want to output standards and zeropoints
1165 self.maxIter = 0
1166 self.outputStandards = True
1167 self.outputZeropoints = True
1168 self.resetFitParameters = False
1169
1170 lutCat = dataRefDict['fgcmLookUpTable'].get()
1171 fgcmLut, lutIndexVals, lutStd = translateFgcmLut(lutCat,
1172 dict(_config.physicalFilterMap))
1173 del lutCat
1174
1175 configDict = makeConfigDict(_config, self.log, camera,
1176 self.maxIter, self.resetFitParameters,
1177 self.outputZeropoints,
1178 lutIndexVals[0]['FILTERNAMES'])
1179
1180 # next we need the exposure/visit information
1181 visitCat = dataRefDict['fgcmVisitCatalog'].get()
1182 fgcmExpInfo = translateVisitCatalog(visitCat)
1183 del visitCat
1184
1185 focalPlaneProjector = FocalPlaneProjector(camera,
1186 self.config.defaultCameraOrientation)
1187
1188 noFitsDict = {'lutIndex': lutIndexVals,
1189 'lutStd': lutStd,
1190 'expInfo': fgcmExpInfo,
1191 'focalPlaneProjector': focalPlaneProjector}
1192
1193 # set up the fitter object
1194 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=False,
1195 noFitsDict=noFitsDict, noOutput=True)
1196
1197 # create the parameter object
1198 if (fgcmFitCycle.initialCycle):
1199 # cycle = 0, initial cycle
1200 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
1201 fgcmLut,
1202 fgcmExpInfo)
1203 else:
1204 if isinstance(dataRefDict['fgcmFitParameters'], afwTable.BaseCatalog):
1205 parCat = dataRefDict['fgcmFitParameters']
1206 else:
1207 parCat = dataRefDict['fgcmFitParameters'].get()
1208 inParInfo, inParams, inSuperStar = self._loadParameters(parCat)
1209 del parCat
1210 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
1211 fgcmExpInfo,
1212 inParInfo,
1213 inParams,
1214 inSuperStar)
1215
1216 # set up the stars...
1217 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
1218
1219 starObs = dataRefDict['fgcmStarObservations'].get()
1220 starIds = dataRefDict['fgcmStarIds'].get()
1221 starIndices = dataRefDict['fgcmStarIndices'].get()
1222
1223 # grab the flagged stars if available
1224 if 'fgcmFlaggedStars' in dataRefDict:
1225 if isinstance(dataRefDict['fgcmFlaggedStars'], afwTable.BaseCatalog):
1226 flaggedStars = dataRefDict['fgcmFlaggedStars']
1227 else:
1228 flaggedStars = dataRefDict['fgcmFlaggedStars'].get()
1229 flagId = flaggedStars['objId'][:]
1230 flagFlag = flaggedStars['objFlag'][:]
1231 else:
1232 flaggedStars = None
1233 flagId = None
1234 flagFlag = None
1235
1236 if _config.doReferenceCalibration:
1237 refStars = dataRefDict['fgcmReferenceStars'].get()
1238
1239 refMag, refMagErr = extractReferenceMags(refStars,
1240 _config.bands,
1241 _config.physicalFilterMap)
1242 refId = refStars['fgcm_id'][:]
1243 else:
1244 refStars = None
1245 refId = None
1246 refMag = None
1247 refMagErr = None
1248
1249 # match star observations to visits
1250 # Only those star observations that match visits from fgcmExpInfo['VISIT'] will
1251 # actually be transferred into fgcm using the indexing below.
1252 visitIndex = np.searchsorted(fgcmExpInfo['VISIT'], starObs['visit'][starIndices['obsIndex']])
1253
1254 # The fgcmStars.loadStars method will copy all the star information into
1255 # special shared memory objects that will not blow up the memory usage when
1256 # used with python multiprocessing. Once all the numbers are copied,
1257 # it is necessary to release all references to the objects that previously
1258 # stored the data to ensure that the garbage collector can clear the memory,
1259 # and ensure that this memory is not copied when multiprocessing kicks in.
1260
1261 # We determine the conversion from the native units (typically radians) to
1262 # degrees for the first star. This allows us to treat coord_ra/coord_dec as
1263 # numpy arrays rather than Angles, which would we approximately 600x slower.
1264 conv = starObs[0]['ra'].asDegrees() / float(starObs[0]['ra'])
1265
1266 fgcmStars.loadStars(fgcmPars,
1267 starObs['visit'][starIndices['obsIndex']],
1268 starObs['ccd'][starIndices['obsIndex']],
1269 starObs['ra'][starIndices['obsIndex']] * conv,
1270 starObs['dec'][starIndices['obsIndex']] * conv,
1271 starObs['instMag'][starIndices['obsIndex']],
1272 starObs['instMagErr'][starIndices['obsIndex']],
1273 fgcmExpInfo['FILTERNAME'][visitIndex],
1274 starIds['fgcm_id'][:],
1275 starIds['ra'][:],
1276 starIds['dec'][:],
1277 starIds['obsArrIndex'][:],
1278 starIds['nObs'][:],
1279 obsX=starObs['x'][starIndices['obsIndex']],
1280 obsY=starObs['y'][starIndices['obsIndex']],
1281 obsDeltaMagBkg=starObs['deltaMagBkg'][starIndices['obsIndex']],
1282 obsDeltaAper=starObs['deltaMagAper'][starIndices['obsIndex']],
1283 psfCandidate=starObs['psf_candidate'][starIndices['obsIndex']],
1284 refID=refId,
1285 refMag=refMag,
1286 refMagErr=refMagErr,
1287 flagID=flagId,
1288 flagFlag=flagFlag,
1289 computeNobs=True)
1290
1291 # Release all references to temporary objects holding star data (see above)
1292 del starObs
1293 del starIds
1294 del starIndices
1295 del flagId
1296 del flagFlag
1297 del flaggedStars
1298 del refStars
1299 del refId
1300 del refMag
1301 del refMagErr
1302
1303 # and set the bits in the cycle object
1304 fgcmFitCycle.setLUT(fgcmLut)
1305 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
1306 fgcmFitCycle.setPars(fgcmPars)
1307
1308 # finish the setup
1309 fgcmFitCycle.finishSetup()
1310
1311 # and run
1312 fgcmFitCycle.run()
1313
1314
1317
1318 fgcmDatasetDict = self._makeFgcmOutputDatasets(fgcmFitCycle)
1319
1320 # Output the config for the next cycle
1321 # We need to make a copy since the input one has been frozen
1322
1323 updatedPhotometricCutDict = {b: float(fgcmFitCycle.updatedPhotometricCut[i]) for
1324 i, b in enumerate(_config.bands)}
1325 updatedHighCutDict = {band: float(fgcmFitCycle.updatedHighCut[i]) for
1326 i, band in enumerate(_config.bands)}
1327
1328 outConfig = copy.copy(_config)
1329 outConfig.update(cycleNumber=(_config.cycleNumber + 1),
1330 precomputeSuperStarInitialCycle=False,
1331 freezeStdAtmosphere=False,
1332 expGrayPhotometricCutDict=updatedPhotometricCutDict,
1333 expGrayHighCutDict=updatedHighCutDict)
1334
1335 outConfig.connections.update(previousCycleNumber=str(_config.cycleNumber),
1336 cycleNumber=str(_config.cycleNumber + 1))
1337
1338 configFileName = '%s_cycle%02d_config.py' % (outConfig.outfileBase,
1339 outConfig.cycleNumber)
1340 outConfig.save(configFileName)
1341
1342 if _config.isFinalCycle == 1:
1343 # We are done, ready to output products
1344 self.log.info("Everything is in place to run fgcmOutputProducts.py")
1345 else:
1346 self.log.info("Saved config for next cycle to %s" % (configFileName))
1347 self.log.info("Be sure to look at:")
1348 self.log.info(" config.expGrayPhotometricCut")
1349 self.log.info(" config.expGrayHighCut")
1350 self.log.info("If you are satisfied with the fit, please set:")
1351 self.log.info(" config.isFinalCycle = True")
1352
1353 fgcmFitCycle.freeSharedMemory()
1354
1355 return fgcmDatasetDict, outConfig
1356
1357 def _checkDatasetsExist(self, butler):
1358 """
1359 Check if necessary datasets exist to run fgcmFitCycle
1360
1361 Parameters
1362 ----------
1363 butler: `lsst.daf.persistence.Butler`
1364
1365 Raises
1366 ------
1367 RuntimeError
1368 If any of fgcmVisitCatalog, fgcmStarObservations, fgcmStarIds,
1369 fgcmStarIndices, fgcmLookUpTable datasets do not exist.
1370 If cycleNumber > 0, then also checks for fgcmFitParameters,
1371 fgcmFlaggedStars.
1372 """
1373
1374 if not butler.datasetExists('fgcmVisitCatalog'):
1375 raise RuntimeError("Could not find fgcmVisitCatalog in repo!")
1376 if not butler.datasetExists('fgcmStarObservations'):
1377 raise RuntimeError("Could not find fgcmStarObservations in repo!")
1378 if not butler.datasetExists('fgcmStarIds'):
1379 raise RuntimeError("Could not find fgcmStarIds in repo!")
1380 if not butler.datasetExists('fgcmStarIndices'):
1381 raise RuntimeError("Could not find fgcmStarIndices in repo!")
1382 if not butler.datasetExists('fgcmLookUpTable'):
1383 raise RuntimeError("Could not find fgcmLookUpTable in repo!")
1384
1385 # Need additional datasets if we are not the initial cycle
1386 if (self.config.cycleNumber > 0):
1387 if not butler.datasetExists('fgcmFitParameters',
1388 fgcmcycle=self.config.cycleNumber-1):
1389 raise RuntimeError("Could not find fgcmFitParameters for previous cycle (%d) in repo!" %
1390 (self.config.cycleNumber-1))
1391 if not butler.datasetExists('fgcmFlaggedStars',
1392 fgcmcycle=self.config.cycleNumber-1):
1393 raise RuntimeError("Could not find fgcmFlaggedStars for previous cycle (%d) in repo!" %
1394 (self.config.cycleNumber-1))
1395
1396 # And additional dataset if we want reference calibration
1397 if self.config.doReferenceCalibration:
1398 if not butler.datasetExists('fgcmReferenceStars'):
1399 raise RuntimeError("Could not find fgcmReferenceStars in repo, and "
1400 "doReferenceCalibration is True.")
1401
1402 def _loadParameters(self, parCat):
1403 """
1404 Load FGCM parameters from a previous fit cycle
1405
1406 Parameters
1407 ----------
1408 parCat : `lsst.afw.table.BaseCatalog`
1409 Parameter catalog in afw table form.
1410
1411 Returns
1412 -------
1413 inParInfo: `numpy.ndarray`
1414 Numpy array parameter information formatted for input to fgcm
1415 inParameters: `numpy.ndarray`
1416 Numpy array parameter values formatted for input to fgcm
1417 inSuperStar: `numpy.array`
1418 Superstar flat formatted for input to fgcm
1419 """
1420 parLutFilterNames = np.array(parCat[0]['lutFilterNames'].split(','))
1421 parFitBands = np.array(parCat[0]['fitBands'].split(','))
1422
1423 inParInfo = np.zeros(1, dtype=[('NCCD', 'i4'),
1424 ('LUTFILTERNAMES', parLutFilterNames.dtype.str,
1425 (parLutFilterNames.size, )),
1426 ('FITBANDS', parFitBands.dtype.str, (parFitBands.size, )),
1427 ('LNTAUUNIT', 'f8'),
1428 ('LNTAUSLOPEUNIT', 'f8'),
1429 ('ALPHAUNIT', 'f8'),
1430 ('LNPWVUNIT', 'f8'),
1431 ('LNPWVSLOPEUNIT', 'f8'),
1432 ('LNPWVQUADRATICUNIT', 'f8'),
1433 ('LNPWVGLOBALUNIT', 'f8'),
1434 ('O3UNIT', 'f8'),
1435 ('QESYSUNIT', 'f8'),
1436 ('FILTEROFFSETUNIT', 'f8'),
1437 ('HASEXTERNALPWV', 'i2'),
1438 ('HASEXTERNALTAU', 'i2')])
1439 inParInfo['NCCD'] = parCat['nCcd']
1440 inParInfo['LUTFILTERNAMES'][:] = parLutFilterNames
1441 inParInfo['FITBANDS'][:] = parFitBands
1442 inParInfo['HASEXTERNALPWV'] = parCat['hasExternalPwv']
1443 inParInfo['HASEXTERNALTAU'] = parCat['hasExternalTau']
1444
1445 inParams = np.zeros(1, dtype=[('PARALPHA', 'f8', (parCat['parAlpha'].size, )),
1446 ('PARO3', 'f8', (parCat['parO3'].size, )),
1447 ('PARLNTAUINTERCEPT', 'f8',
1448 (parCat['parLnTauIntercept'].size, )),
1449 ('PARLNTAUSLOPE', 'f8',
1450 (parCat['parLnTauSlope'].size, )),
1451 ('PARLNPWVINTERCEPT', 'f8',
1452 (parCat['parLnPwvIntercept'].size, )),
1453 ('PARLNPWVSLOPE', 'f8',
1454 (parCat['parLnPwvSlope'].size, )),
1455 ('PARLNPWVQUADRATIC', 'f8',
1456 (parCat['parLnPwvQuadratic'].size, )),
1457 ('PARQESYSINTERCEPT', 'f8',
1458 (parCat['parQeSysIntercept'].size, )),
1459 ('COMPQESYSSLOPE', 'f8',
1460 (parCat['compQeSysSlope'].size, )),
1461 ('PARFILTEROFFSET', 'f8',
1462 (parCat['parFilterOffset'].size, )),
1463 ('PARFILTEROFFSETFITFLAG', 'i2',
1464 (parCat['parFilterOffsetFitFlag'].size, )),
1465 ('PARRETRIEVEDLNPWVSCALE', 'f8'),
1466 ('PARRETRIEVEDLNPWVOFFSET', 'f8'),
1467 ('PARRETRIEVEDLNPWVNIGHTLYOFFSET', 'f8',
1468 (parCat['parRetrievedLnPwvNightlyOffset'].size, )),
1469 ('COMPABSTHROUGHPUT', 'f8',
1470 (parCat['compAbsThroughput'].size, )),
1471 ('COMPREFOFFSET', 'f8',
1472 (parCat['compRefOffset'].size, )),
1473 ('COMPREFSIGMA', 'f8',
1474 (parCat['compRefSigma'].size, )),
1475 ('COMPMIRRORCHROMATICITY', 'f8',
1476 (parCat['compMirrorChromaticity'].size, )),
1477 ('MIRRORCHROMATICITYPIVOT', 'f8',
1478 (parCat['mirrorChromaticityPivot'].size, )),
1479 ('COMPMEDIANSEDSLOPE', 'f8',
1480 (parCat['compMedianSedSlope'].size, )),
1481 ('COMPAPERCORRPIVOT', 'f8',
1482 (parCat['compAperCorrPivot'].size, )),
1483 ('COMPAPERCORRSLOPE', 'f8',
1484 (parCat['compAperCorrSlope'].size, )),
1485 ('COMPAPERCORRSLOPEERR', 'f8',
1486 (parCat['compAperCorrSlopeErr'].size, )),
1487 ('COMPAPERCORRRANGE', 'f8',
1488 (parCat['compAperCorrRange'].size, )),
1489 ('COMPMODELERREXPTIMEPIVOT', 'f8',
1490 (parCat['compModelErrExptimePivot'].size, )),
1491 ('COMPMODELERRFWHMPIVOT', 'f8',
1492 (parCat['compModelErrFwhmPivot'].size, )),
1493 ('COMPMODELERRSKYPIVOT', 'f8',
1494 (parCat['compModelErrSkyPivot'].size, )),
1495 ('COMPMODELERRPARS', 'f8',
1496 (parCat['compModelErrPars'].size, )),
1497 ('COMPEXPGRAY', 'f8',
1498 (parCat['compExpGray'].size, )),
1499 ('COMPVARGRAY', 'f8',
1500 (parCat['compVarGray'].size, )),
1501 ('COMPEXPDELTAMAGBKG', 'f8',
1502 (parCat['compExpDeltaMagBkg'].size, )),
1503 ('COMPNGOODSTARPEREXP', 'i4',
1504 (parCat['compNGoodStarPerExp'].size, )),
1505 ('COMPSIGFGCM', 'f8',
1506 (parCat['compSigFgcm'].size, )),
1507 ('COMPSIGMACAL', 'f8',
1508 (parCat['compSigmaCal'].size, )),
1509 ('COMPRETRIEVEDLNPWV', 'f8',
1510 (parCat['compRetrievedLnPwv'].size, )),
1511 ('COMPRETRIEVEDLNPWVRAW', 'f8',
1512 (parCat['compRetrievedLnPwvRaw'].size, )),
1513 ('COMPRETRIEVEDLNPWVFLAG', 'i2',
1514 (parCat['compRetrievedLnPwvFlag'].size, )),
1515 ('COMPRETRIEVEDTAUNIGHT', 'f8',
1516 (parCat['compRetrievedTauNight'].size, )),
1517 ('COMPEPSILON', 'f8',
1518 (parCat['compEpsilon'].size, )),
1519 ('COMPMEDDELTAAPER', 'f8',
1520 (parCat['compMedDeltaAper'].size, )),
1521 ('COMPGLOBALEPSILON', 'f4',
1522 (parCat['compGlobalEpsilon'].size, )),
1523 ('COMPEPSILONMAP', 'f4',
1524 (parCat['compEpsilonMap'].size, )),
1525 ('COMPEPSILONNSTARMAP', 'i4',
1526 (parCat['compEpsilonNStarMap'].size, )),
1527 ('COMPEPSILONCCDMAP', 'f4',
1528 (parCat['compEpsilonCcdMap'].size, )),
1529 ('COMPEPSILONCCDNSTARMAP', 'i4',
1530 (parCat['compEpsilonCcdNStarMap'].size, ))])
1531
1532 inParams['PARALPHA'][:] = parCat['parAlpha'][0, :]
1533 inParams['PARO3'][:] = parCat['parO3'][0, :]
1534 inParams['PARLNTAUINTERCEPT'][:] = parCat['parLnTauIntercept'][0, :]
1535 inParams['PARLNTAUSLOPE'][:] = parCat['parLnTauSlope'][0, :]
1536 inParams['PARLNPWVINTERCEPT'][:] = parCat['parLnPwvIntercept'][0, :]
1537 inParams['PARLNPWVSLOPE'][:] = parCat['parLnPwvSlope'][0, :]
1538 inParams['PARLNPWVQUADRATIC'][:] = parCat['parLnPwvQuadratic'][0, :]
1539 inParams['PARQESYSINTERCEPT'][:] = parCat['parQeSysIntercept'][0, :]
1540 inParams['COMPQESYSSLOPE'][:] = parCat['compQeSysSlope'][0, :]
1541 inParams['PARFILTEROFFSET'][:] = parCat['parFilterOffset'][0, :]
1542 inParams['PARFILTEROFFSETFITFLAG'][:] = parCat['parFilterOffsetFitFlag'][0, :]
1543 inParams['PARRETRIEVEDLNPWVSCALE'] = parCat['parRetrievedLnPwvScale']
1544 inParams['PARRETRIEVEDLNPWVOFFSET'] = parCat['parRetrievedLnPwvOffset']
1545 inParams['PARRETRIEVEDLNPWVNIGHTLYOFFSET'][:] = parCat['parRetrievedLnPwvNightlyOffset'][0, :]
1546 inParams['COMPABSTHROUGHPUT'][:] = parCat['compAbsThroughput'][0, :]
1547 inParams['COMPREFOFFSET'][:] = parCat['compRefOffset'][0, :]
1548 inParams['COMPREFSIGMA'][:] = parCat['compRefSigma'][0, :]
1549 inParams['COMPMIRRORCHROMATICITY'][:] = parCat['compMirrorChromaticity'][0, :]
1550 inParams['MIRRORCHROMATICITYPIVOT'][:] = parCat['mirrorChromaticityPivot'][0, :]
1551 inParams['COMPMEDIANSEDSLOPE'][:] = parCat['compMedianSedSlope'][0, :]
1552 inParams['COMPAPERCORRPIVOT'][:] = parCat['compAperCorrPivot'][0, :]
1553 inParams['COMPAPERCORRSLOPE'][:] = parCat['compAperCorrSlope'][0, :]
1554 inParams['COMPAPERCORRSLOPEERR'][:] = parCat['compAperCorrSlopeErr'][0, :]
1555 inParams['COMPAPERCORRRANGE'][:] = parCat['compAperCorrRange'][0, :]
1556 inParams['COMPMODELERREXPTIMEPIVOT'][:] = parCat['compModelErrExptimePivot'][0, :]
1557 inParams['COMPMODELERRFWHMPIVOT'][:] = parCat['compModelErrFwhmPivot'][0, :]
1558 inParams['COMPMODELERRSKYPIVOT'][:] = parCat['compModelErrSkyPivot'][0, :]
1559 inParams['COMPMODELERRPARS'][:] = parCat['compModelErrPars'][0, :]
1560 inParams['COMPEXPGRAY'][:] = parCat['compExpGray'][0, :]
1561 inParams['COMPVARGRAY'][:] = parCat['compVarGray'][0, :]
1562 inParams['COMPEXPDELTAMAGBKG'][:] = parCat['compExpDeltaMagBkg'][0, :]
1563 inParams['COMPNGOODSTARPEREXP'][:] = parCat['compNGoodStarPerExp'][0, :]
1564 inParams['COMPSIGFGCM'][:] = parCat['compSigFgcm'][0, :]
1565 inParams['COMPSIGMACAL'][:] = parCat['compSigmaCal'][0, :]
1566 inParams['COMPRETRIEVEDLNPWV'][:] = parCat['compRetrievedLnPwv'][0, :]
1567 inParams['COMPRETRIEVEDLNPWVRAW'][:] = parCat['compRetrievedLnPwvRaw'][0, :]
1568 inParams['COMPRETRIEVEDLNPWVFLAG'][:] = parCat['compRetrievedLnPwvFlag'][0, :]
1569 inParams['COMPRETRIEVEDTAUNIGHT'][:] = parCat['compRetrievedTauNight'][0, :]
1570 inParams['COMPEPSILON'][:] = parCat['compEpsilon'][0, :]
1571 inParams['COMPMEDDELTAAPER'][:] = parCat['compMedDeltaAper'][0, :]
1572 inParams['COMPGLOBALEPSILON'][:] = parCat['compGlobalEpsilon'][0, :]
1573 inParams['COMPEPSILONMAP'][:] = parCat['compEpsilonMap'][0, :]
1574 inParams['COMPEPSILONNSTARMAP'][:] = parCat['compEpsilonNStarMap'][0, :]
1575 inParams['COMPEPSILONCCDMAP'][:] = parCat['compEpsilonCcdMap'][0, :]
1576 inParams['COMPEPSILONCCDNSTARMAP'][:] = parCat['compEpsilonCcdNStarMap'][0, :]
1577
1578 inSuperStar = np.zeros(parCat['superstarSize'][0, :], dtype='f8')
1579 inSuperStar[:, :, :, :] = parCat['superstar'][0, :].reshape(inSuperStar.shape)
1580
1581 return (inParInfo, inParams, inSuperStar)
1582
1583 def _makeFgcmOutputDatasets(self, fgcmFitCycle):
1584 """
1585 Persist FGCM datasets through the butler.
1586
1587 Parameters
1588 ----------
1589 fgcmFitCycle: `lsst.fgcm.FgcmFitCycle`
1590 Fgcm Fit cycle object
1591 """
1592 fgcmDatasetDict = {}
1593
1594 # Save the parameters
1595 parInfo, pars = fgcmFitCycle.fgcmPars.parsToArrays()
1596
1597 parSchema = afwTable.Schema()
1598
1599 comma = ','
1600 lutFilterNameString = comma.join([n.decode('utf-8')
1601 for n in parInfo['LUTFILTERNAMES'][0]])
1602 fitBandString = comma.join([n.decode('utf-8')
1603 for n in parInfo['FITBANDS'][0]])
1604
1605 parSchema = self._makeParSchema(parInfo, pars, fgcmFitCycle.fgcmPars.parSuperStarFlat,
1606 lutFilterNameString, fitBandString)
1607 parCat = self._makeParCatalog(parSchema, parInfo, pars,
1608 fgcmFitCycle.fgcmPars.parSuperStarFlat,
1609 lutFilterNameString, fitBandString)
1610
1611 fgcmDatasetDict['fgcmFitParameters'] = parCat
1612
1613 # Save the indices of the flagged stars
1614 # (stars that have been (a) reserved from the fit for testing and
1615 # (b) bad stars that have failed quality checks.)
1616 flagStarSchema = self._makeFlagStarSchema()
1617 flagStarStruct = fgcmFitCycle.fgcmStars.getFlagStarIndices()
1618 flagStarCat = self._makeFlagStarCat(flagStarSchema, flagStarStruct)
1619
1620 fgcmDatasetDict['fgcmFlaggedStars'] = flagStarCat
1621
1622 # Save the zeropoint information and atmospheres only if desired
1623 if self.outputZeropoints:
1624 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_SSTAR_CHEB'].shape[1]
1625 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct['FGCM_FZPT_CHEB'].shape[1]
1626
1627 zptSchema = makeZptSchema(superStarChebSize, zptChebSize)
1628 zptCat = makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
1629
1630 fgcmDatasetDict['fgcmZeropoints'] = zptCat
1631
1632 # Save atmosphere values
1633 # These are generated by the same code that generates zeropoints
1634 atmSchema = makeAtmSchema()
1635 atmCat = makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
1636
1637 fgcmDatasetDict['fgcmAtmosphereParameters'] = atmCat
1638
1639 # Save the standard stars (if configured)
1640 if self.outputStandards:
1641 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
1642 stdSchema = makeStdSchema(len(goodBands))
1643 stdCat = makeStdCat(stdSchema, stdStruct, goodBands)
1644
1645 fgcmDatasetDict['fgcmStandardStars'] = stdCat
1646
1647 return fgcmDatasetDict
1648
1649 def _makeParSchema(self, parInfo, pars, parSuperStarFlat,
1650 lutFilterNameString, fitBandString):
1651 """
1652 Make the parameter persistence schema
1653
1654 Parameters
1655 ----------
1656 parInfo: `numpy.ndarray`
1657 Parameter information returned by fgcm
1658 pars: `numpy.ndarray`
1659 Parameter values returned by fgcm
1660 parSuperStarFlat: `numpy.array`
1661 Superstar flat values returned by fgcm
1662 lutFilterNameString: `str`
1663 Combined string of all the lutFilterNames
1664 fitBandString: `str`
1665 Combined string of all the fitBands
1666
1667 Returns
1668 -------
1669 parSchema: `afwTable.schema`
1670 """
1671
1672 parSchema = afwTable.Schema()
1673
1674 # parameter info section
1675 parSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
1676 parSchema.addField('lutFilterNames', type=str, doc='LUT Filter names in parameter file',
1677 size=len(lutFilterNameString))
1678 parSchema.addField('fitBands', type=str, doc='Bands that were fit',
1679 size=len(fitBandString))
1680 parSchema.addField('lnTauUnit', type=np.float64, doc='Step units for ln(AOD)')
1681 parSchema.addField('lnTauSlopeUnit', type=np.float64,
1682 doc='Step units for ln(AOD) slope')
1683 parSchema.addField('alphaUnit', type=np.float64, doc='Step units for alpha')
1684 parSchema.addField('lnPwvUnit', type=np.float64, doc='Step units for ln(pwv)')
1685 parSchema.addField('lnPwvSlopeUnit', type=np.float64,
1686 doc='Step units for ln(pwv) slope')
1687 parSchema.addField('lnPwvQuadraticUnit', type=np.float64,
1688 doc='Step units for ln(pwv) quadratic term')
1689 parSchema.addField('lnPwvGlobalUnit', type=np.float64,
1690 doc='Step units for global ln(pwv) parameters')
1691 parSchema.addField('o3Unit', type=np.float64, doc='Step units for O3')
1692 parSchema.addField('qeSysUnit', type=np.float64, doc='Step units for mirror gray')
1693 parSchema.addField('filterOffsetUnit', type=np.float64, doc='Step units for filter offset')
1694 parSchema.addField('hasExternalPwv', type=np.int32, doc='Parameters fit using external pwv')
1695 parSchema.addField('hasExternalTau', type=np.int32, doc='Parameters fit using external tau')
1696
1697 # parameter section
1698 parSchema.addField('parAlpha', type='ArrayD', doc='Alpha parameter vector',
1699 size=pars['PARALPHA'].size)
1700 parSchema.addField('parO3', type='ArrayD', doc='O3 parameter vector',
1701 size=pars['PARO3'].size)
1702 parSchema.addField('parLnTauIntercept', type='ArrayD',
1703 doc='ln(Tau) intercept parameter vector',
1704 size=pars['PARLNTAUINTERCEPT'].size)
1705 parSchema.addField('parLnTauSlope', type='ArrayD',
1706 doc='ln(Tau) slope parameter vector',
1707 size=pars['PARLNTAUSLOPE'].size)
1708 parSchema.addField('parLnPwvIntercept', type='ArrayD', doc='ln(pwv) intercept parameter vector',
1709 size=pars['PARLNPWVINTERCEPT'].size)
1710 parSchema.addField('parLnPwvSlope', type='ArrayD', doc='ln(pwv) slope parameter vector',
1711 size=pars['PARLNPWVSLOPE'].size)
1712 parSchema.addField('parLnPwvQuadratic', type='ArrayD', doc='ln(pwv) quadratic parameter vector',
1713 size=pars['PARLNPWVQUADRATIC'].size)
1714 parSchema.addField('parQeSysIntercept', type='ArrayD', doc='Mirror gray intercept parameter vector',
1715 size=pars['PARQESYSINTERCEPT'].size)
1716 parSchema.addField('compQeSysSlope', type='ArrayD', doc='Mirror gray slope parameter vector',
1717 size=pars[0]['COMPQESYSSLOPE'].size)
1718 parSchema.addField('parFilterOffset', type='ArrayD', doc='Filter offset parameter vector',
1719 size=pars['PARFILTEROFFSET'].size)
1720 parSchema.addField('parFilterOffsetFitFlag', type='ArrayI', doc='Filter offset parameter fit flag',
1721 size=pars['PARFILTEROFFSETFITFLAG'].size)
1722 parSchema.addField('parRetrievedLnPwvScale', type=np.float64,
1723 doc='Global scale for retrieved ln(pwv)')
1724 parSchema.addField('parRetrievedLnPwvOffset', type=np.float64,
1725 doc='Global offset for retrieved ln(pwv)')
1726 parSchema.addField('parRetrievedLnPwvNightlyOffset', type='ArrayD',
1727 doc='Nightly offset for retrieved ln(pwv)',
1728 size=pars['PARRETRIEVEDLNPWVNIGHTLYOFFSET'].size)
1729 parSchema.addField('compAbsThroughput', type='ArrayD',
1730 doc='Absolute throughput (relative to transmission curves)',
1731 size=pars['COMPABSTHROUGHPUT'].size)
1732 parSchema.addField('compRefOffset', type='ArrayD',
1733 doc='Offset between reference stars and calibrated stars',
1734 size=pars['COMPREFOFFSET'].size)
1735 parSchema.addField('compRefSigma', type='ArrayD',
1736 doc='Width of reference star/calibrated star distribution',
1737 size=pars['COMPREFSIGMA'].size)
1738 parSchema.addField('compMirrorChromaticity', type='ArrayD',
1739 doc='Computed mirror chromaticity terms',
1740 size=pars['COMPMIRRORCHROMATICITY'].size)
1741 parSchema.addField('mirrorChromaticityPivot', type='ArrayD',
1742 doc='Mirror chromaticity pivot mjd',
1743 size=pars['MIRRORCHROMATICITYPIVOT'].size)
1744 parSchema.addField('compMedianSedSlope', type='ArrayD',
1745 doc='Computed median SED slope (per band)',
1746 size=pars['COMPMEDIANSEDSLOPE'].size)
1747 parSchema.addField('compAperCorrPivot', type='ArrayD', doc='Aperture correction pivot',
1748 size=pars['COMPAPERCORRPIVOT'].size)
1749 parSchema.addField('compAperCorrSlope', type='ArrayD', doc='Aperture correction slope',
1750 size=pars['COMPAPERCORRSLOPE'].size)
1751 parSchema.addField('compAperCorrSlopeErr', type='ArrayD', doc='Aperture correction slope error',
1752 size=pars['COMPAPERCORRSLOPEERR'].size)
1753 parSchema.addField('compAperCorrRange', type='ArrayD', doc='Aperture correction range',
1754 size=pars['COMPAPERCORRRANGE'].size)
1755 parSchema.addField('compModelErrExptimePivot', type='ArrayD', doc='Model error exptime pivot',
1756 size=pars['COMPMODELERREXPTIMEPIVOT'].size)
1757 parSchema.addField('compModelErrFwhmPivot', type='ArrayD', doc='Model error fwhm pivot',
1758 size=pars['COMPMODELERRFWHMPIVOT'].size)
1759 parSchema.addField('compModelErrSkyPivot', type='ArrayD', doc='Model error sky pivot',
1760 size=pars['COMPMODELERRSKYPIVOT'].size)
1761 parSchema.addField('compModelErrPars', type='ArrayD', doc='Model error parameters',
1762 size=pars['COMPMODELERRPARS'].size)
1763 parSchema.addField('compExpGray', type='ArrayD', doc='Computed exposure gray',
1764 size=pars['COMPEXPGRAY'].size)
1765 parSchema.addField('compVarGray', type='ArrayD', doc='Computed exposure variance',
1766 size=pars['COMPVARGRAY'].size)
1767 parSchema.addField('compExpDeltaMagBkg', type='ArrayD',
1768 doc='Computed exposure offset due to background',
1769 size=pars['COMPEXPDELTAMAGBKG'].size)
1770 parSchema.addField('compNGoodStarPerExp', type='ArrayI',
1771 doc='Computed number of good stars per exposure',
1772 size=pars['COMPNGOODSTARPEREXP'].size)
1773 parSchema.addField('compSigFgcm', type='ArrayD', doc='Computed sigma_fgcm (intrinsic repeatability)',
1774 size=pars['COMPSIGFGCM'].size)
1775 parSchema.addField('compSigmaCal', type='ArrayD', doc='Computed sigma_cal (systematic error floor)',
1776 size=pars['COMPSIGMACAL'].size)
1777 parSchema.addField('compRetrievedLnPwv', type='ArrayD', doc='Retrieved ln(pwv) (smoothed)',
1778 size=pars['COMPRETRIEVEDLNPWV'].size)
1779 parSchema.addField('compRetrievedLnPwvRaw', type='ArrayD', doc='Retrieved ln(pwv) (raw)',
1780 size=pars['COMPRETRIEVEDLNPWVRAW'].size)
1781 parSchema.addField('compRetrievedLnPwvFlag', type='ArrayI', doc='Retrieved ln(pwv) Flag',
1782 size=pars['COMPRETRIEVEDLNPWVFLAG'].size)
1783 parSchema.addField('compRetrievedTauNight', type='ArrayD', doc='Retrieved tau (per night)',
1784 size=pars['COMPRETRIEVEDTAUNIGHT'].size)
1785 parSchema.addField('compEpsilon', type='ArrayD',
1786 doc='Computed epsilon background offset per visit (nJy/arcsec2)',
1787 size=pars['COMPEPSILON'].size)
1788 parSchema.addField('compMedDeltaAper', type='ArrayD',
1789 doc='Median delta mag aper per visit',
1790 size=pars['COMPMEDDELTAAPER'].size)
1791 parSchema.addField('compGlobalEpsilon', type='ArrayD',
1792 doc='Computed epsilon bkg offset (global) (nJy/arcsec2)',
1793 size=pars['COMPGLOBALEPSILON'].size)
1794 parSchema.addField('compEpsilonMap', type='ArrayD',
1795 doc='Computed epsilon maps (nJy/arcsec2)',
1796 size=pars['COMPEPSILONMAP'].size)
1797 parSchema.addField('compEpsilonNStarMap', type='ArrayI',
1798 doc='Number of stars per pixel in computed epsilon maps',
1799 size=pars['COMPEPSILONNSTARMAP'].size)
1800 parSchema.addField('compEpsilonCcdMap', type='ArrayD',
1801 doc='Computed epsilon ccd maps (nJy/arcsec2)',
1802 size=pars['COMPEPSILONCCDMAP'].size)
1803 parSchema.addField('compEpsilonCcdNStarMap', type='ArrayI',
1804 doc='Number of stars per ccd bin in epsilon ccd maps',
1805 size=pars['COMPEPSILONCCDNSTARMAP'].size)
1806 # superstarflat section
1807 parSchema.addField('superstarSize', type='ArrayI', doc='Superstar matrix size',
1808 size=4)
1809 parSchema.addField('superstar', type='ArrayD', doc='Superstar matrix (flattened)',
1810 size=parSuperStarFlat.size)
1811
1812 return parSchema
1813
1814 def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
1815 lutFilterNameString, fitBandString):
1816 """
1817 Make the FGCM parameter catalog for persistence
1818
1819 Parameters
1820 ----------
1821 parSchema: `lsst.afw.table.Schema`
1822 Parameter catalog schema
1823 pars: `numpy.ndarray`
1824 FGCM parameters to put into parCat
1825 parSuperStarFlat: `numpy.array`
1826 FGCM superstar flat array to put into parCat
1827 lutFilterNameString: `str`
1828 Combined string of all the lutFilterNames
1829 fitBandString: `str`
1830 Combined string of all the fitBands
1831
1832 Returns
1833 -------
1834 parCat: `afwTable.BasicCatalog`
1835 Atmosphere and instrumental model parameter catalog for persistence
1836 """
1837
1838 parCat = afwTable.BaseCatalog(parSchema)
1839 parCat.reserve(1)
1840
1841 # The parameter catalog just has one row, with many columns for all the
1842 # atmosphere and instrument fit parameters
1843 rec = parCat.addNew()
1844
1845 # info section
1846 rec['nCcd'] = parInfo['NCCD']
1847 rec['lutFilterNames'] = lutFilterNameString
1848 rec['fitBands'] = fitBandString
1849 # note these are not currently supported here.
1850 rec['hasExternalPwv'] = 0
1851 rec['hasExternalTau'] = 0
1852
1853 # parameter section
1854
1855 scalarNames = ['parRetrievedLnPwvScale', 'parRetrievedLnPwvOffset']
1856
1857 arrNames = ['parAlpha', 'parO3', 'parLnTauIntercept', 'parLnTauSlope',
1858 'parLnPwvIntercept', 'parLnPwvSlope', 'parLnPwvQuadratic',
1859 'parQeSysIntercept', 'compQeSysSlope',
1860 'parRetrievedLnPwvNightlyOffset', 'compAperCorrPivot',
1861 'parFilterOffset', 'parFilterOffsetFitFlag',
1862 'compAbsThroughput', 'compRefOffset', 'compRefSigma',
1863 'compMirrorChromaticity', 'mirrorChromaticityPivot',
1864 'compAperCorrSlope', 'compAperCorrSlopeErr', 'compAperCorrRange',
1865 'compModelErrExptimePivot', 'compModelErrFwhmPivot',
1866 'compModelErrSkyPivot', 'compModelErrPars',
1867 'compExpGray', 'compVarGray', 'compNGoodStarPerExp', 'compSigFgcm',
1868 'compSigmaCal', 'compExpDeltaMagBkg', 'compMedianSedSlope',
1869 'compRetrievedLnPwv', 'compRetrievedLnPwvRaw', 'compRetrievedLnPwvFlag',
1870 'compRetrievedTauNight', 'compEpsilon', 'compMedDeltaAper',
1871 'compGlobalEpsilon', 'compEpsilonMap', 'compEpsilonNStarMap',
1872 'compEpsilonCcdMap', 'compEpsilonCcdNStarMap']
1873
1874 for scalarName in scalarNames:
1875 rec[scalarName] = pars[scalarName.upper()]
1876
1877 for arrName in arrNames:
1878 rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]
1879
1880 # superstar section
1881 rec['superstarSize'][:] = parSuperStarFlat.shape
1882 rec['superstar'][:] = parSuperStarFlat.ravel()
1883
1884 return parCat
1885
1886 def _makeFlagStarSchema(self):
1887 """
1888 Make the flagged-stars schema
1889
1890 Returns
1891 -------
1892 flagStarSchema: `lsst.afw.table.Schema`
1893 """
1894
1895 flagStarSchema = afwTable.Schema()
1896
1897 flagStarSchema.addField('objId', type=np.int32, doc='FGCM object id')
1898 flagStarSchema.addField('objFlag', type=np.int32, doc='FGCM object flag')
1899
1900 return flagStarSchema
1901
1902 def _makeFlagStarCat(self, flagStarSchema, flagStarStruct):
1903 """
1904 Make the flagged star catalog for persistence
1905
1906 Parameters
1907 ----------
1908 flagStarSchema: `lsst.afw.table.Schema`
1909 Flagged star schema
1910 flagStarStruct: `numpy.ndarray`
1911 Flagged star structure from fgcm
1912
1913 Returns
1914 -------
1915 flagStarCat: `lsst.afw.table.BaseCatalog`
1916 Flagged star catalog for persistence
1917 """
1918
1919 flagStarCat = afwTable.BaseCatalog(flagStarSchema)
1920 flagStarCat.resize(flagStarStruct.size)
1921
1922 flagStarCat['objId'][:] = flagStarStruct['OBJID']
1923 flagStarCat['objFlag'][:] = flagStarStruct['OBJFLAG']
1924
1925 return flagStarCat
def extractReferenceMags(refStars, bands, filterMap)
Definition: utilities.py:901
def makeStdSchema(nBands)
Definition: utilities.py:752
def makeAtmCat(atmSchema, atmStruct)
Definition: utilities.py:719
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, lutFilterNames, tract=None)
Definition: utilities.py:52
def translateFgcmLut(lutCat, physicalFilterMap)
Definition: utilities.py:230
def makeZptCat(zptSchema, zpStruct)
Definition: utilities.py:640
def makeStdCat(stdSchema, stdStruct, goodBands)
Definition: utilities.py:787
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
Definition: utilities.py:554
def translateVisitCatalog(visitCat)
Definition: utilities.py:360